diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml new file mode 100755 index 000000000000..9ad815875900 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index 877a01b4c107..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_array(_airbyte_data, "$['conflict_stream_array']") as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index 13acc814aa35..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` --- conflict_stream_array -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index b0a10f06f6fe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(array_to_string(conflict_stream_array, "|", "") as - string -), '')) as - string -))) as _airbyte_conflict_stream_array_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` tmp --- conflict_stream_array -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index e32dbb75a795..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - - json_extract(table_alias._airbyte_data, "$['conflict_stream_name']") - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index 1e6066d334e2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(conflict_stream_name as - string -) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` --- conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index f8867d6b5713..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(conflict_stream_name as - string -), '')) as - string -))) as _airbyte_conflict_stream_name_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` tmp --- conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index 57cef056c421..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - json_extract(table_alias.conflict_stream_name, "$['conflict_stream_name']") - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`conflict_stream_name` as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index eae8466d7725..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as - string -) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index 0ea122e2ea01..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as - string -), ''), '-', coalesce(cast(conflict_stream_name as - string -), '')) as - string -))) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index 962511141a3b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - json_extract_scalar(conflict_stream_name, "$['groups']") as `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index be894b9a62a3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(`groups` as - string -) as `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index c468c76f6378..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as - string -), ''), '-', coalesce(cast(`groups` as - string -), '')) as - string -))) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index c933d2c41fac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['conflict_stream_scalar']") as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index f1c973037920..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(conflict_stream_scalar as - int64 -) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` --- conflict_stream_scalar -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 2679d90727ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(conflict_stream_scalar as - string -), '')) as - string -))) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp --- conflict_stream_scalar -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql deleted file mode 100644 index 19a85c1c7d63..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['date']") as date, - - json_extract(table_alias._airbyte_data, "$['partition']") - as `partition`, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_complex_columns_resulting_into_long_names -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql deleted file mode 100644 index a7d06c7427b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,24 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(date as - string -) as date, - cast(`partition` as - string -) as `partition`, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` --- nested_stream_with_complex_columns_resulting_into_long_names -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql deleted file mode 100644 index a6fa2d3eba18..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_extract_scalar(DATA, "$['currency']") as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -cross join unnest(DATA) as DATA -where 1 = 1 -and DATA is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql deleted file mode 100644 index 1e4b82a39dec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - string -) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql deleted file mode 100644 index c60a413f2d05..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as - string -), ''), '-', coalesce(cast(currency as - string -), '')) as - string -))) as _airbyte_DATA_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql deleted file mode 100644 index 556ff2471ce9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - json_extract_array(`partition`, "$['double_array_data']") as double_array_data, - json_extract_array(`partition`, "$['DATA']") as DATA, - json_extract_array(`partition`, "$['column___with__quotes']") as column___with__quotes, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and `partition` is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql deleted file mode 100644 index 0dbb6ecbb1f9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - DATA, - column___with__quotes, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql deleted file mode 100644 index 56fd4a34bd7f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as - string -), ''), '-', coalesce(cast(array_to_string(double_array_data, "|", "") as - string -), ''), '-', coalesce(cast(array_to_string(DATA, "|", "") as - string -), ''), '-', coalesce(cast(array_to_string(column___with__quotes, "|", "") as - string -), '')) as - string -))) as _airbyte_partition_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql deleted file mode 100644 index e7ab56070330..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_extract_scalar(column___with__quotes, "$['currency']") as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -cross join unnest(column___with__quotes) as column___with__quotes -where 1 = 1 -and column___with__quotes is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql deleted file mode 100644 index b552081271d7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - string -) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql deleted file mode 100644 index cdc26222cfab..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as - string -), ''), '-', coalesce(cast(currency as - string -), '')) as - string -))) as _airbyte_column___with__quotes_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql deleted file mode 100644 index 18469ff3ada9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_extract_scalar(double_array_data, "$['id']") as id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -cross join unnest(double_array_data) as double_array_data -where 1 = 1 -and double_array_data is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql deleted file mode 100644 index bc3564b761f1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as - string -) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql deleted file mode 100644 index 410d26ac7a07..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as - string -), ''), '-', coalesce(cast(id as - string -), '')) as - string -))) as _airbyte_double_array_data_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql deleted file mode 100644 index eae11107c950..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['date']") as date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql deleted file mode 100644 index 3006d17a73e9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(date as - string -) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql deleted file mode 100644 index 707db214afcd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(date as - string -), '')) as - string -))) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` tmp --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 043cd72e489d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_array(_airbyte_data, "$['children']") as children, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index e99a33441a66..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` --- unnest_alias -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index d807cce078ad..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(array_to_string(children, "|", "") as - string -), '')) as - string -))) as _airbyte_unnest_alias_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` tmp --- unnest_alias -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 9896f675369f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_unnest_alias_hashid, - json_extract_scalar(children, "$['ab_id']") as ab_id, - - json_extract(children, "$['owner']") - as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`unnest_alias` as table_alias --- children at unnest_alias/children -cross join unnest(children) as children -where 1 = 1 -and children is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index 4347ac1d1e98..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - int64 -) as ab_id, - cast(owner as - string -) as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` --- children at unnest_alias/children -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index 07b1fea81aa2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as - string -), ''), '-', coalesce(cast(ab_id as - string -), ''), '-', coalesce(cast(owner as - string -), '')) as - string -))) as _airbyte_children_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` tmp --- children at unnest_alias/children -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 0af940c468cf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - json_extract_scalar(owner, "$['owner_id']") as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`unnest_alias_children` as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and owner is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 1d59670f9820..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - int64 -) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` --- owner at unnest_alias/children/owner -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 4c1f50bc18a4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_children_hashid as - string -), ''), '-', coalesce(cast(owner_id as - string -), '')) as - string -))) as _airbyte_owner_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp --- owner at unnest_alias/children/owner -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql deleted file mode 100644 index 7d71f4199706..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['date']") as date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql deleted file mode 100644 index 51c4df2f7fba..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(date as - string -) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql deleted file mode 100644 index b86eed32466f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(date as - string -), '')) as - string -))) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` tmp --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 11c5f0e7d7c9..4997112c84b4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -5,7 +5,7 @@ _airbyte_active_row, generate_array(0, 1, 1) ) - cluster by _airbyte_unique_key, _airbyte_emitted_at + cluster by _airbyte_unique_key_scd, _airbyte_emitted_at OPTIONS() as ( @@ -36,13 +36,13 @@ scd_data as ( date desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(date) over ( + case when row_number() over ( partition by id order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..6fccd15a86bd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,70 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + json_extract_array(`partition`, "$['double_array_data']") as double_array_data, + json_extract_array(`partition`, "$['DATA']") as DATA, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + DATA, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as + string +), ''), '-', coalesce(cast(array_to_string(double_array_data, "|", "") as + string +), ''), '-', coalesce(cast(array_to_string(DATA, "|", "") as + string +), '')) as + string +))) as _airbyte_partition_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + DATA, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_partition_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql new file mode 100644 index 000000000000..705cafb3cce4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql @@ -0,0 +1,69 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(DATA, "$['currency']") as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join unnest(DATA) as DATA +where 1 = 1 +and DATA is not null + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + string +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(currency as + string +), '')) as + string +))) as _airbyte_DATA_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2 tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_DATA_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..e409ece448db --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,69 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(double_array_data, "$['id']") as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join unnest(double_array_data) as double_array_data +where 1 = 1 +and double_array_data is not null + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as + string +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_double_array_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index a3e70d30888d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab3` --- unnest_alias from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias -where 1 = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index f641bb360a1b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_array` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` --- conflict_stream_array from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index 2a7669945831..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` --- conflict_stream_name from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 1d897d05e549..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 033f337977b9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index b5b99cd91982..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_scalar` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` --- conflict_stream_scalar from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index 9e5aca960cfd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - DATA, - column___with__quotes, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_partition_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql deleted file mode 100644 index d8f8943f5149..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_DATA_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql deleted file mode 100644 index 01a20f04da09..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index 639fa690b35b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql deleted file mode 100644 index 880063feec3c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` --- non_nested_stream_without_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index b0aa1b41329f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_children_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` --- children at unnest_alias/children from `dataline-integration-testing`.test_normalization.`unnest_alias` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index 4be0121256bd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children_owner` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_owner_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` --- owner at unnest_alias/children/owner from `dataline-integration-testing`.test_normalization.`unnest_alias_children` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql deleted file mode 100644 index b2236adf2322..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid -from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` --- simple_stream_with_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql index 36e9afe3d413..76d046456ddd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -3,7 +3,43 @@ create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab3` OPTIONS() as --- SQL model to build a hash column based on the values of this record +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + + json_extract(table_alias._airbyte_data, "$['partition']") + as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + cast(`partition` as + string +) as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select to_hex(md5(cast(concat(coalesce(cast(id as string @@ -15,7 +51,7 @@ select string ))) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` tmp +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 tmp -- nested_stream_with_complex_columns_resulting_into_long_names where 1 = 1 ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index 17c06b46b9de..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index b9f5ca00d5f1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_array_ab1') }} --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index 925451397e03..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - array_to_string('conflict_stream_array'), - ]) }} as _airbyte_conflict_stream_array_hashid, - tmp.* -from {{ ref('conflict_stream_array_ab2') }} tmp --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index 850e03194fc5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index 1d6d46e0d1ff..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_ab1') }} --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index 1ea9d02a7136..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'conflict_stream_name', - ]) }} as _airbyte_conflict_stream_name_hashid, - tmp.* -from {{ ref('conflict_stream_name_ab2') }} tmp --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index e6c77c8bc197..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name') }} as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index 80bebf6d45b6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index eaec60ab1634..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_conflict_stream_name_hashid', - 'conflict_stream_name', - ]) }} as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index 917c7f8d18ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as {{ adapter.quote('groups') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index f20eec57368b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast({{ adapter.quote('groups') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('groups') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index 67fab3b94f9e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_conflict_stream_name_2_hashid', - adapter.quote('groups'), - ]) }} as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2') }} tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index eb1e66fa8ee3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index b66d0d244bec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_scalar_ab1') }} --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 18e6e00c4681..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'conflict_stream_scalar', - ]) }} as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from {{ ref('conflict_stream_scalar_ab2') }} tmp --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql index 2f9195fcd8f1..792f469a3403 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -1,7 +1,7 @@ {{ config( cluster_by = "_airbyte_emitted_at", partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql index d8f527d720d4..31c8c52d5152 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -1,7 +1,7 @@ {{ config( cluster_by = "_airbyte_emitted_at", partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql index 1e77394f833f..6a577541f762 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql @@ -1,7 +1,6 @@ {{ config( cluster_by = "_airbyte_emitted_at", partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -18,4 +17,5 @@ from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partit {{ cross_join_unnest('partition', 'DATA') }} where 1 = 1 and DATA is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql deleted file mode 100644 index d069f1a86923..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1') }} --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql deleted file mode 100644 index 856be375a7cc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'currency', - ]) }} as _airbyte_DATA_hashid, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2') }} tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql index b5cd1da52bd6..f77dea0a88ac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -1,7 +1,7 @@ {{ config( cluster_by = "_airbyte_emitted_at", partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -10,12 +10,12 @@ select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as DATA, - {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as column___with__quotes, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} as table_alias -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 and {{ adapter.quote('partition') }} is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql deleted file mode 100644 index 23d645909627..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - DATA, - column___with__quotes, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1') }} --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql deleted file mode 100644 index c77aba11220e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid', - array_to_string('double_array_data'), - array_to_string('DATA'), - array_to_string('column___with__quotes'), - ]) }} as _airbyte_partition_hashid, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2') }} tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql deleted file mode 100644 index 4cbc027f3f71..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'column___with__quotes') }} -select - _airbyte_partition_hashid, - {{ json_extract_scalar(unnested_column_value('column___with__quotes'), ['currency'], ['currency']) }} as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -{{ cross_join_unnest('partition', 'column___with__quotes') }} -where 1 = 1 -and column___with__quotes is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql deleted file mode 100644 index 8a107f04e7e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1') }} --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql deleted file mode 100644 index 59b1e25f08fb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'currency', - ]) }} as _airbyte_column___with__quotes_hashid, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2') }} tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql index e7e798b05822..eb8226613e16 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -1,7 +1,6 @@ {{ config( cluster_by = "_airbyte_emitted_at", partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -18,4 +17,5 @@ from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partit {{ cross_join_unnest('partition', 'double_array_data') }} where 1 = 1 and double_array_data is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql deleted file mode 100644 index 034a47f9c894..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1') }} --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql deleted file mode 100644 index 60a4c34970ac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'id', - ]) }} as _airbyte_double_array_data_hashid, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2') }} tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql deleted file mode 100644 index ed2d3cd69d1d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql deleted file mode 100644 index 671850765c83..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab1') }} --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql deleted file mode 100644 index 15f30e6226cf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'date', - ]) }} as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, - tmp.* -from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab2') }} tmp --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 5a136fe032bb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index afc783083e6c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_ab1') }} --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index ce3a0718103d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - array_to_string('children'), - ]) }} as _airbyte_unnest_alias_hashid, - tmp.* -from {{ ref('unnest_alias_ab2') }} tmp --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 995c1a2bdb48..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} -select - _airbyte_unnest_alias_hashid, - {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, - {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias') }} as table_alias --- children at unnest_alias/children -{{ cross_join_unnest('unnest_alias', 'children') }} -where 1 = 1 -and children is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index 0cb0d82d585c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, - cast(owner as {{ type_json() }}) as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children_ab1') }} --- children at unnest_alias/children -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index 5110b59063a1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_unnest_alias_hashid', - 'ab_id', - 'owner', - ]) }} as _airbyte_children_hashid, - tmp.* -from {{ ref('unnest_alias_children_ab2') }} tmp --- children at unnest_alias/children -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index edda3677624d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - {{ json_extract_scalar('owner', ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children') }} as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and owner is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 5bc8a3f96b2c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children_owner_ab1') }} --- owner at unnest_alias/children/owner -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 16753dc228c5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_children_hashid', - 'owner_id', - ]) }} as _airbyte_owner_hashid, - tmp.* -from {{ ref('unnest_alias_children_owner_ab2') }} tmp --- owner at unnest_alias/children/owner -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql deleted file mode 100644 index f29ea4161b7d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql deleted file mode 100644 index 11e2b2ec37bd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab1') }} --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql deleted file mode 100644 index 79e6653964c0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'date', - ]) }} as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, - tmp.* -from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab2') }} tmp --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index b2389c8fd05d..0de83f0e8463 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -1,5 +1,5 @@ {{ config( - cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + cluster_by = ["_airbyte_unique_key_scd","_airbyte_emitted_at"], partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", @@ -64,13 +64,13 @@ scd_data as ( date desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(date) over ( + case when row_number() over ( partition by id order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..f8fa44ff7306 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,21 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + DATA, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_partition_hashid +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3') }} +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql new file mode 100644 index 000000000000..f9c5510c20ac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql @@ -0,0 +1,19 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_DATA_hashid +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3') }} +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..440c888d4b55 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,19 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3') }} +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index 1c38bf472728..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index fd790b1b5f1a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from {{ ref('conflict_stream_array_ab3') }} --- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index def7986d8580..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from {{ ref('conflict_stream_name_ab3') }} --- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 539ff778ebdf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 1f43352fa617..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - {{ adapter.quote('groups') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index a013b69e57aa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from {{ ref('conflict_stream_scalar_ab3') }} --- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index ef861e9fa13e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - DATA, - column___with__quotes, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_partition_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3') }} --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql deleted file mode 100644 index 7b461f4989e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_DATA_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3') }} --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql deleted file mode 100644 index 2bcc0ec32e51..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3') }} --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index 353362155623..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3') }} --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql deleted file mode 100644 index 14c598a1568d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid -from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab3') }} --- non_nested_stream_without_namespace_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 7e78f023a44e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_children_hashid -from {{ ref('unnest_alias_children_ab3') }} --- children at unnest_alias/children from {{ ref('unnest_alias') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index c81e06c620b1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_owner_hashid -from {{ ref('unnest_alias_children_owner_ab3') }} --- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql deleted file mode 100644 index 54dec00c79ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization_namespace", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid -from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab3') }} --- simple_stream_with_namespace_resulting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql index f4835b370e3c..82431353bc2b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -1,7 +1,7 @@ {{ config( cluster_by = "_airbyte_emitted_at", partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index 877a01b4c107..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_array(_airbyte_data, "$['conflict_stream_array']") as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index 13acc814aa35..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` --- conflict_stream_array -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index b0a10f06f6fe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(array_to_string(conflict_stream_array, "|", "") as - string -), '')) as - string -))) as _airbyte_conflict_stream_array_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` tmp --- conflict_stream_array -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index e32dbb75a795..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - - json_extract(table_alias._airbyte_data, "$['conflict_stream_name']") - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index 1e6066d334e2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(conflict_stream_name as - string -) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` --- conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index f8867d6b5713..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(conflict_stream_name as - string -), '')) as - string -))) as _airbyte_conflict_stream_name_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` tmp --- conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index 57cef056c421..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - json_extract(table_alias.conflict_stream_name, "$['conflict_stream_name']") - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`conflict_stream_name` as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index eae8466d7725..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as - string -) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index 0ea122e2ea01..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as - string -), ''), '-', coalesce(cast(conflict_stream_name as - string -), '')) as - string -))) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index 962511141a3b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - json_extract_scalar(conflict_stream_name, "$['groups']") as `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index be894b9a62a3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(`groups` as - string -) as `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index c468c76f6378..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as - string -), ''), '-', coalesce(cast(`groups` as - string -), '')) as - string -))) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index c933d2c41fac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['conflict_stream_scalar']") as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index f1c973037920..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(conflict_stream_scalar as - int64 -) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` --- conflict_stream_scalar -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 2679d90727ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(conflict_stream_scalar as - string -), '')) as - string -))) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp --- conflict_stream_scalar -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql deleted file mode 100644 index 19a85c1c7d63..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['date']") as date, - - json_extract(table_alias._airbyte_data, "$['partition']") - as `partition`, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_complex_columns_resulting_into_long_names -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql deleted file mode 100644 index a7d06c7427b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,24 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(date as - string -) as date, - cast(`partition` as - string -) as `partition`, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` --- nested_stream_with_complex_columns_resulting_into_long_names -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql deleted file mode 100644 index a6fa2d3eba18..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_extract_scalar(DATA, "$['currency']") as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -cross join unnest(DATA) as DATA -where 1 = 1 -and DATA is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql deleted file mode 100644 index 1e4b82a39dec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - string -) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql deleted file mode 100644 index c60a413f2d05..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as - string -), ''), '-', coalesce(cast(currency as - string -), '')) as - string -))) as _airbyte_DATA_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql deleted file mode 100644 index 556ff2471ce9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - json_extract_array(`partition`, "$['double_array_data']") as double_array_data, - json_extract_array(`partition`, "$['DATA']") as DATA, - json_extract_array(`partition`, "$['column___with__quotes']") as column___with__quotes, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and `partition` is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql deleted file mode 100644 index 0dbb6ecbb1f9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - DATA, - column___with__quotes, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql deleted file mode 100644 index 56fd4a34bd7f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as - string -), ''), '-', coalesce(cast(array_to_string(double_array_data, "|", "") as - string -), ''), '-', coalesce(cast(array_to_string(DATA, "|", "") as - string -), ''), '-', coalesce(cast(array_to_string(column___with__quotes, "|", "") as - string -), '')) as - string -))) as _airbyte_partition_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql deleted file mode 100644 index e7ab56070330..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_extract_scalar(column___with__quotes, "$['currency']") as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -cross join unnest(column___with__quotes) as column___with__quotes -where 1 = 1 -and column___with__quotes is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql deleted file mode 100644 index b552081271d7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - string -) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql deleted file mode 100644 index cdc26222cfab..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as - string -), ''), '-', coalesce(cast(currency as - string -), '')) as - string -))) as _airbyte_column___with__quotes_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql deleted file mode 100644 index 18469ff3ada9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_extract_scalar(double_array_data, "$['id']") as id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -cross join unnest(double_array_data) as double_array_data -where 1 = 1 -and double_array_data is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql deleted file mode 100644 index bc3564b761f1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as - string -) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql deleted file mode 100644 index 410d26ac7a07..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as - string -), ''), '-', coalesce(cast(id as - string -), '')) as - string -))) as _airbyte_double_array_data_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql deleted file mode 100644 index eae11107c950..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['date']") as date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql deleted file mode 100644 index 3006d17a73e9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(date as - string -) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql deleted file mode 100644 index 707db214afcd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(date as - string -), '')) as - string -))) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` tmp --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 043cd72e489d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_array(_airbyte_data, "$['children']") as children, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index e99a33441a66..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` --- unnest_alias -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index d807cce078ad..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(array_to_string(children, "|", "") as - string -), '')) as - string -))) as _airbyte_unnest_alias_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` tmp --- unnest_alias -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 9896f675369f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_unnest_alias_hashid, - json_extract_scalar(children, "$['ab_id']") as ab_id, - - json_extract(children, "$['owner']") - as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`unnest_alias` as table_alias --- children at unnest_alias/children -cross join unnest(children) as children -where 1 = 1 -and children is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index 4347ac1d1e98..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - int64 -) as ab_id, - cast(owner as - string -) as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` --- children at unnest_alias/children -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index 07b1fea81aa2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as - string -), ''), '-', coalesce(cast(ab_id as - string -), ''), '-', coalesce(cast(owner as - string -), '')) as - string -))) as _airbyte_children_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` tmp --- children at unnest_alias/children -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 0af940c468cf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - json_extract_scalar(owner, "$['owner_id']") as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization.`unnest_alias_children` as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and owner is not null; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 1d59670f9820..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - int64 -) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` --- owner at unnest_alias/children/owner -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 4c1f50bc18a4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(_airbyte_children_hashid as - string -), ''), '-', coalesce(cast(owner_id as - string -), '')) as - string -))) as _airbyte_owner_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp --- owner at unnest_alias/children/owner -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql deleted file mode 100644 index 7d71f4199706..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['date']") as date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql deleted file mode 100644 index 51c4df2f7fba..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - string -) as id, - cast(date as - string -) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql deleted file mode 100644 index b86eed32466f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(date as - string -), '')) as - string -))) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` tmp --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..fbb4d9e00550 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_ab_id = DBT_INTERNAL_DEST._airbyte_ab_id + + + + when matched then update set + `_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`,`double_array_data` = DBT_INTERNAL_SOURCE.`double_array_data`,`DATA` = DBT_INTERNAL_SOURCE.`DATA`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_partition_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_partition_hashid` + + + when not matched then insert + (`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`, `double_array_data`, `DATA`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_partition_hashid`) + values + (`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`, `double_array_data`, `DATA`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_partition_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql new file mode 100644 index 000000000000..da77d8e6172f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql @@ -0,0 +1,21 @@ + + + + + + + merge into `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on FALSE + + + + when not matched then insert + (`_airbyte_partition_hashid`, `currency`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_DATA_hashid`) + values + (`_airbyte_partition_hashid`, `currency`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_DATA_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..a1198af2586c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,21 @@ + + + + + + + merge into `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on FALSE + + + + when not matched then insert + (`_airbyte_partition_hashid`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_double_array_data_hashid`) + values + (`_airbyte_partition_hashid`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_double_array_data_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index 683fbd323fdb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - merge into `dataline-integration-testing`.test_normalization.`unnest_alias` as DBT_INTERNAL_DEST - using ( - select * from `dataline-integration-testing`.test_normalization.`unnest_alias__dbt_tmp` - ) as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._airbyte_ab_id = DBT_INTERNAL_DEST._airbyte_ab_id - - - - when matched then update set - `id` = DBT_INTERNAL_SOURCE.`id`,`children` = DBT_INTERNAL_SOURCE.`children`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_unnest_alias_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_unnest_alias_hashid` - - - when not matched then insert - (`id`, `children`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_unnest_alias_hashid`) - values - (`id`, `children`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_unnest_alias_hashid`) - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index f641bb360a1b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_array` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` --- conflict_stream_array from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index 2a7669945831..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` --- conflict_stream_name from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 1d897d05e549..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 033f337977b9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index b5b99cd91982..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_scalar` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` --- conflict_stream_scalar from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index 9e5aca960cfd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - DATA, - column___with__quotes, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_partition_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql deleted file mode 100644 index d8f8943f5149..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_DATA_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql deleted file mode 100644 index 01a20f04da09..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index 639fa690b35b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql deleted file mode 100644 index 880063feec3c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` --- non_nested_stream_without_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index b0aa1b41329f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_children_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` --- children at unnest_alias/children from `dataline-integration-testing`.test_normalization.`unnest_alias` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index 4be0121256bd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children_owner` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_owner_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` --- owner at unnest_alias/children/owner from `dataline-integration-testing`.test_normalization.`unnest_alias_children` -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql deleted file mode 100644 index b2236adf2322..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid -from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` --- simple_stream_with_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql index 36e9afe3d413..76d046456ddd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -3,7 +3,43 @@ create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab3` OPTIONS() as --- SQL model to build a hash column based on the values of this record +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + + json_extract(table_alias._airbyte_data, "$['partition']") + as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + cast(`partition` as + string +) as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select to_hex(md5(cast(concat(coalesce(cast(id as string @@ -15,7 +51,7 @@ select string ))) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` tmp +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 tmp -- nested_stream_with_complex_columns_resulting_into_long_names where 1 = 1 ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml new file mode 100755 index 000000000000..522b1e595e8a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["modified_models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml new file mode 100644 index 000000000000..9ad815875900 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index eecd7c33da6d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['name']") as name, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 995249ea89b1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,30 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(name as - string -) as name, - cast(_ab_cdc_lsn as - float64 -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float64 -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float64 -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` --- dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index aba44bffd591..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['currency']") as currency, - json_extract_scalar(_airbyte_data, "$['date']") as date, - json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, - json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, - json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, - json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, - json_extract_scalar(_airbyte_data, "$['USD']") as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index 37dc10cde3ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,39 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(currency as - string -) as currency, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp -) as timestamp_col, - cast(HKD_special___characters as - float64 -) as HKD_special___characters, - cast(HKD_special___characters_1 as - string -) as HKD_special___characters_1, - cast(NZD as - float64 -) as NZD, - cast(USD as - float64 -) as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` --- dedup_exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 7e55806a2d74..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['currency']") as currency, - json_extract_scalar(_airbyte_data, "$['date']") as date, - json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, - json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, - json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, - json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, - json_extract_scalar(_airbyte_data, "$['USD']") as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 2bfffd7d4b36..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,39 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(currency as - string -) as currency, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp -) as timestamp_col, - cast(HKD_special___characters as - float64 -) as HKD_special___characters, - cast(HKD_special___characters_1 as - string -) as HKD_special___characters_1, - cast(NZD as - float64 -) as NZD, - cast(USD as - float64 -) as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` --- exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index 4c66cad39bd6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,32 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(currency as - string -), ''), '-', coalesce(cast(date as - string -), ''), '-', coalesce(cast(timestamp_col as - string -), ''), '-', coalesce(cast(HKD_special___characters as - string -), ''), '-', coalesce(cast(HKD_special___characters_1 as - string -), ''), '-', coalesce(cast(NZD as - string -), ''), '-', coalesce(cast(USD as - string -), '')) as - string -))) as _airbyte_exchange_rate_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` tmp --- exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 267f6e648459..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['name']") as name, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_log_pos']") as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index 58262d8eaaa5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,32 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(name as - string -) as name, - cast(_ab_cdc_lsn as - float64 -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float64 -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float64 -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float64 -) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` --- pos_dedup_cdcx -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 9c881dde8e67..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index b427d44957c8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` --- renamed_dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 4b61673a2f50..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,98 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` - partition by range_bucket( - _airbyte_active_row, - generate_array(0, 1, 1) - ) - cluster by _airbyte_unique_key, _airbyte_emitted_at - OPTIONS() - as ( - -with - -input_data as ( - select * - from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` - -- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), '')) as - string -))) as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as - string -), cast(_ab_cdc_updated_at as - string -) - order by _airbyte_ab_id - ) as _airbyte_row_num, - to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as - string -), ''), '-', coalesce(cast(_airbyte_start_at as - string -), ''), '-', coalesce(cast(_airbyte_emitted_at as - string -), ''), '-', coalesce(cast(_ab_cdc_deleted_at as - string -), ''), '-', coalesce(cast(_ab_cdc_updated_at as - string -), '')) as - string -))) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index f999aac2ea61..0a20ae9875ed 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -5,7 +5,7 @@ _airbyte_active_row, generate_array(0, 1, 1) ) - cluster by _airbyte_unique_key, _airbyte_emitted_at + cluster by _airbyte_unique_key_scd, _airbyte_emitted_at OPTIONS() as ( @@ -47,7 +47,7 @@ scd_data as ( date desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(date) over ( + case when row_number() over ( partition by id, currency, cast(NZD as string ) @@ -55,7 +55,7 @@ scd_data as ( date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 12494cc91c84..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,82 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` - partition by range_bucket( - _airbyte_active_row, - generate_array(0, 1, 1) - ) - cluster by _airbyte_unique_key, _airbyte_emitted_at - OPTIONS() - as ( - -with - -input_data as ( - select * - from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` - -- renamed_dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), '')) as - string -))) as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_renamed_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id - ) as _airbyte_row_num, - to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as - string -), ''), '-', coalesce(cast(_airbyte_start_at as - string -), ''), '-', coalesce(cast(_airbyte_emitted_at as - string -), '')) as - string -))) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 79eebde9c581..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_unique_key, _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` --- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded -where 1 = 1 -and _airbyte_active_row = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index c49e28422cd3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,28 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`exchange_rate` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` --- exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate -where 1 = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 1b67b9d0c597..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_unique_key, _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` --- renamed_dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded -where 1 = 1 -and _airbyte_active_row = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 20c879115c65..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,104 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` - partition by range_bucket( - _airbyte_active_row, - generate_array(0, 1, 1) - ) - cluster by _airbyte_unique_key, _airbyte_emitted_at - OPTIONS() - as ( - -with - -input_data as ( - select * - from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` - -- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), '')) as - string -))) as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as - string -), cast(_ab_cdc_updated_at as - string -), cast(_ab_cdc_log_pos as - string -) - order by _airbyte_ab_id - ) as _airbyte_row_num, - to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as - string -), ''), '-', coalesce(cast(_airbyte_start_at as - string -), ''), '-', coalesce(cast(_airbyte_emitted_at as - string -), ''), '-', coalesce(cast(_ab_cdc_deleted_at as - string -), ''), '-', coalesce(cast(_ab_cdc_updated_at as - string -), ''), '-', coalesce(cast(_ab_cdc_log_pos as - string -), '')) as - string -))) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..fa29a805cce1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,113 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`exchange_rate` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +with __dbt__cte__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + json_extract_scalar(_airbyte_data, "$['column___with__quotes']") as column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(currency as + string +) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(HKD_special___characters_1 as + string +) as HKD_special___characters_1, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + cast(column___with__quotes as + string +) as column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(HKD_special___characters_1 as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), ''), '-', coalesce(cast(column___with__quotes as + string +), '')) as + string +))) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__cte__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__cte__exchange_rate_ab3 +-- exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index bde8bc2db495..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_unique_key, _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` --- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and _airbyte_active_row = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 662067add2e0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(name as - string -), ''), '-', coalesce(cast(_ab_cdc_lsn as - string -), ''), '-', coalesce(cast(_ab_cdc_updated_at as - string -), ''), '-', coalesce(cast(_ab_cdc_deleted_at as - string -), '')) as - string -))) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp --- dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 4ed21e656fe2..c676cab9183c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -3,7 +3,61 @@ create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` OPTIONS() as --- SQL model to build a hash column based on the values of this record +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(currency as + string +) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(HKD_special___characters_1 as + string +) as HKD_special___characters_1, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select to_hex(md5(cast(concat(coalesce(cast(id as string @@ -25,7 +79,7 @@ select string ))) as _airbyte_dedup_exchange_rate_hashid, tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +from __dbt__cte__dedup_exchange_rate_ab2 tmp -- dedup_exchange_rate where 1 = 1 ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 4936ef6153fe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(name as - string -), ''), '-', coalesce(cast(_ab_cdc_lsn as - string -), ''), '-', coalesce(cast(_ab_cdc_updated_at as - string -), ''), '-', coalesce(cast(_ab_cdc_deleted_at as - string -), ''), '-', coalesce(cast(_ab_cdc_log_pos as - string -), '')) as - string -))) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp --- pos_dedup_cdcx -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 016b91d7bded..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), '')) as - string -))) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp --- renamed_dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index fa3068bcaa9f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 322f1de599d7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(name as {{ dbt_utils.type_string() }}) as name, - cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('dedup_cdc_excluded_ab1') }} --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index a1925d6ed311..75209f7ca54e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,7 +1,7 @@ {{ config( cluster_by = "_airbyte_emitted_at", partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 3790a0629706..d77248246190 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,7 +1,7 @@ {{ config( cluster_by = "_airbyte_emitted_at", partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 51af17b7e5c1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, - {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as HKD_special___characters, - {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as HKD_special___characters_1, - {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, - {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 319bc81f8575..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - cast({{ empty_string_to_null('date') }} as {{ type_date() }}) as date, - cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col, - cast(HKD_special___characters as {{ dbt_utils.type_float() }}) as HKD_special___characters, - cast(HKD_special___characters_1 as {{ dbt_utils.type_string() }}) as HKD_special___characters_1, - cast(NZD as {{ dbt_utils.type_float() }}) as NZD, - cast(USD as {{ dbt_utils.type_float() }}) as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('exchange_rate_ab1') }} --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index 0c6628f131eb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'date', - 'timestamp_col', - 'HKD_special___characters', - 'HKD_special___characters_1', - 'NZD', - 'USD', - ]) }} as _airbyte_exchange_rate_hashid, - tmp.* -from {{ ref('exchange_rate_ab2') }} tmp --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index b69171ca3358..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index 40c6e7e9aa85..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(name as {{ dbt_utils.type_string() }}) as name, - cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('pos_dedup_cdcx_ab1') }} --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 221619f8f03c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index c9cd56d5d915..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('renamed_dedup_cdc_excluded_ab1') }} --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 69b881ca1ed9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,113 +0,0 @@ -{{ config( - cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], - partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('dedup_cdc_excluded_ab3') }} - -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('dedup_cdc_excluded_ab3') }} - -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index aff096b2e6f1..7192b0ad0168 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,5 +1,5 @@ {{ config( - cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + cluster_by = ["_airbyte_unique_key_scd","_airbyte_emitted_at"], partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", @@ -73,13 +73,13 @@ scd_data as ( date desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(date) over ( + case when row_number() over ( partition by id, currency, cast(NZD as {{ dbt_utils.type_string() }}) order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index f008ba4b8401..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,105 +0,0 @@ -{{ config( - cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], - partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} - -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} - -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_renamed_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 14050eec8255..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config( - cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where 1 = 1 -and _airbyte_active_row = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index c579db34bb15..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 75212a4d2aa8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from {{ ref('renamed_dedup_cdc_excluded_scd') }} --- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} -where 1 = 1 -and _airbyte_active_row = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 58db901dc577..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,115 +0,0 @@ -{{ config( - cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], - partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..9396aa59b921 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,26 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index fe75b24654a8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config( - cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_scd') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where 1 = 1 -and _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 46ac10d781f8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 2b0dd30ab3d7..4595975e6630 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,7 +1,7 @@ {{ config( cluster_by = "_airbyte_emitted_at", partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index ee3dc14c336c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 905b8f4c4087..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = "_airbyte_emitted_at", - partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..8ffdb01ce40c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,25 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, + {{ json_extract_scalar('_airbyte_data', ['new_column'], ['new_column']) }} as new_column, + {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, + {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, + {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as HKD_special___characters, + {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, + {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..960b00900a9d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,25 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_float() }}) as id, + cast(currency as {{ dbt_utils.type_string() }}) as currency, + cast(new_column as {{ dbt_utils.type_float() }}) as new_column, + cast({{ empty_string_to_null('date') }} as {{ type_date() }}) as date, + cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col, + cast(HKD_special___characters as {{ dbt_utils.type_float() }}) as HKD_special___characters, + cast(NZD as {{ dbt_utils.type_float() }}) as NZD, + cast(USD as {{ dbt_utils.type_bigint() }}) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('dedup_exchange_rate_ab1') }} +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..1ff3a08768e6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,123 @@ +{{ config( + cluster_by = ["_airbyte_unique_key_scd","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'NZD', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'NZD', + ]) }} as _airbyte_unique_key, + id, + currency, + new_column, + date, + timestamp_col, + HKD_special___characters, + NZD, + USD, + date as _airbyte_start_at, + lag(date) over ( + partition by cast(id as {{ dbt_utils.type_string() }}), currency, cast(NZD as {{ dbt_utils.type_string() }}) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when row_number() over ( + partition by cast(id as {{ dbt_utils.type_string() }}), currency, cast(NZD as {{ dbt_utils.type_string() }}) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) = 1 then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + new_column, + date, + timestamp_col, + HKD_special___characters, + NZD, + USD, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..3c496db01893 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,28 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + new_column, + date, + timestamp_col, + HKD_special___characters, + NZD, + USD, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from {{ ref('dedup_exchange_rate_scd') }} +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..1206e85ce7df --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,26 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + new_column, + date, + timestamp_col, + HKD_special___characters, + NZD, + USD, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..f7525275b926 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,25 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'new_column', + 'date', + 'timestamp_col', + 'HKD_special___characters', + 'NZD', + 'USD', + ]) }} as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/sources.yml new file mode 100644 index 000000000000..dd538a80131a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/sources.yml @@ -0,0 +1,11 @@ +version: 2 +sources: +- name: test_normalization + quoting: + database: true + schema: false + identifier: false + tables: + - name: _airbyte_raw_dedup_exchange_rate + - name: _airbyte_raw_exchange_rate + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index eecd7c33da6d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['name']") as name, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 995249ea89b1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,30 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(name as - string -) as name, - cast(_ab_cdc_lsn as - float64 -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float64 -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float64 -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` --- dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index aba44bffd591..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['currency']") as currency, - json_extract_scalar(_airbyte_data, "$['date']") as date, - json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, - json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, - json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, - json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, - json_extract_scalar(_airbyte_data, "$['USD']") as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index 37dc10cde3ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,39 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(currency as - string -) as currency, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp -) as timestamp_col, - cast(HKD_special___characters as - float64 -) as HKD_special___characters, - cast(HKD_special___characters_1 as - string -) as HKD_special___characters_1, - cast(NZD as - float64 -) as NZD, - cast(USD as - float64 -) as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` --- dedup_exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 7e55806a2d74..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['currency']") as currency, - json_extract_scalar(_airbyte_data, "$['date']") as date, - json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, - json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, - json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, - json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, - json_extract_scalar(_airbyte_data, "$['USD']") as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 2bfffd7d4b36..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,39 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(currency as - string -) as currency, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp -) as timestamp_col, - cast(HKD_special___characters as - float64 -) as HKD_special___characters, - cast(HKD_special___characters_1 as - string -) as HKD_special___characters_1, - cast(NZD as - float64 -) as NZD, - cast(USD as - float64 -) as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` --- exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index 4c66cad39bd6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,32 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(currency as - string -), ''), '-', coalesce(cast(date as - string -), ''), '-', coalesce(cast(timestamp_col as - string -), ''), '-', coalesce(cast(HKD_special___characters as - string -), ''), '-', coalesce(cast(HKD_special___characters_1 as - string -), ''), '-', coalesce(cast(NZD as - string -), ''), '-', coalesce(cast(USD as - string -), '')) as - string -))) as _airbyte_exchange_rate_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` tmp --- exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 267f6e648459..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['name']") as name, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_log_pos']") as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index 58262d8eaaa5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,32 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(name as - string -) as name, - cast(_ab_cdc_lsn as - float64 -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float64 -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float64 -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float64 -) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` --- pos_dedup_cdcx -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 9c881dde8e67..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index b427d44957c8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` --- renamed_dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index bef8e97a0a47..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - merge into `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` as DBT_INTERNAL_DEST - using ( - select * from `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` - ) as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd - - - - when matched then update set - `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_dedup_cdc_excluded_hashid` - - - when not matched then insert - (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) - values - (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 83688c333707..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` as DBT_INTERNAL_DEST - using ( - select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` - ) as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd - - - - when matched then update set - `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid` - - - when not matched then insert - (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) - values - (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 5ad7a0737609..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - merge into `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded` as DBT_INTERNAL_DEST - using ( - select * from `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded__dbt_tmp` - ) as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key - - - - when matched then update set - `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_dedup_cdc_excluded_hashid` - - - when not matched then insert - (`_airbyte_unique_key`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) - values - (`_airbyte_unique_key`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index cf06f296aea9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - merge into `dataline-integration-testing`.test_normalization.`exchange_rate` as DBT_INTERNAL_DEST - using ( - select * from `dataline-integration-testing`.test_normalization.`exchange_rate__dbt_tmp` - ) as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._airbyte_ab_id = DBT_INTERNAL_DEST._airbyte_ab_id - - - - when matched then update set - `id` = DBT_INTERNAL_SOURCE.`id`,`currency` = DBT_INTERNAL_SOURCE.`currency`,`date` = DBT_INTERNAL_SOURCE.`date`,`timestamp_col` = DBT_INTERNAL_SOURCE.`timestamp_col`,`HKD_special___characters` = DBT_INTERNAL_SOURCE.`HKD_special___characters`,`HKD_special___characters_1` = DBT_INTERNAL_SOURCE.`HKD_special___characters_1`,`NZD` = DBT_INTERNAL_SOURCE.`NZD`,`USD` = DBT_INTERNAL_SOURCE.`USD`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_exchange_rate_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_exchange_rate_hashid` - - - when not matched then insert - (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `HKD_special___characters_1`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`) - values - (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `HKD_special___characters_1`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`) - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 0a8394cf9abc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded` as DBT_INTERNAL_DEST - using ( - select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` - ) as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key - - - - when matched then update set - `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid` - - - when not matched then insert - (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) - values - (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 20c879115c65..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,104 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` - partition by range_bucket( - _airbyte_active_row, - generate_array(0, 1, 1) - ) - cluster by _airbyte_unique_key, _airbyte_emitted_at - OPTIONS() - as ( - -with - -input_data as ( - select * - from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` - -- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), '')) as - string -))) as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as - string -), cast(_ab_cdc_updated_at as - string -), cast(_ab_cdc_log_pos as - string -) - order by _airbyte_ab_id - ) as _airbyte_row_num, - to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as - string -), ''), '-', coalesce(cast(_airbyte_start_at as - string -), ''), '-', coalesce(cast(_airbyte_emitted_at as - string -), ''), '-', coalesce(cast(_ab_cdc_deleted_at as - string -), ''), '-', coalesce(cast(_ab_cdc_updated_at as - string -), ''), '-', coalesce(cast(_ab_cdc_log_pos as - string -), '')) as - string -))) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..fa29a805cce1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,113 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`exchange_rate` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +with __dbt__cte__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + json_extract_scalar(_airbyte_data, "$['column___with__quotes']") as column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(currency as + string +) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(HKD_special___characters_1 as + string +) as HKD_special___characters_1, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + cast(column___with__quotes as + string +) as column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(HKD_special___characters_1 as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), ''), '-', coalesce(cast(column___with__quotes as + string +), '')) as + string +))) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__cte__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__cte__exchange_rate_ab3 +-- exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index bde8bc2db495..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx` - partition by timestamp_trunc(_airbyte_emitted_at, day) - cluster by _airbyte_unique_key, _airbyte_emitted_at - OPTIONS() - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` --- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and _airbyte_active_row = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 662067add2e0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(name as - string -), ''), '-', coalesce(cast(_ab_cdc_lsn as - string -), ''), '-', coalesce(cast(_ab_cdc_updated_at as - string -), ''), '-', coalesce(cast(_ab_cdc_deleted_at as - string -), '')) as - string -))) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp --- dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 4ed21e656fe2..c676cab9183c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -3,7 +3,61 @@ create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` OPTIONS() as --- SQL model to build a hash column based on the values of this record +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(currency as + string +) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(HKD_special___characters_1 as + string +) as HKD_special___characters_1, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select to_hex(md5(cast(concat(coalesce(cast(id as string @@ -25,7 +79,7 @@ select string ))) as _airbyte_dedup_exchange_rate_hashid, tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +from __dbt__cte__dedup_exchange_rate_ab2 tmp -- dedup_exchange_rate where 1 = 1 ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 4936ef6153fe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(name as - string -), ''), '-', coalesce(cast(_ab_cdc_lsn as - string -), ''), '-', coalesce(cast(_ab_cdc_updated_at as - string -), ''), '-', coalesce(cast(_ab_cdc_deleted_at as - string -), ''), '-', coalesce(cast(_ab_cdc_log_pos as - string -), '')) as - string -))) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp --- pos_dedup_cdcx -where 1 = 1; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 016b91d7bded..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), '')) as - string -))) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp --- renamed_dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index 5efa724e88f2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['currency']") as currency, - json_extract_scalar(_airbyte_data, "$['new_column']") as new_column, - json_extract_scalar(_airbyte_data, "$['date']") as date, - json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, - json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, - json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, - json_extract_scalar(_airbyte_data, "$['USD']") as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index 45be772bd787..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,39 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - float64 -) as id, - cast(currency as - string -) as currency, - cast(new_column as - float64 -) as new_column, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp -) as timestamp_col, - cast(HKD_special___characters as - float64 -) as HKD_special___characters, - cast(NZD as - float64 -) as NZD, - cast(USD as - int64 -) as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` --- dedup_exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index cceb45e4340f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['currency']") as currency, - json_extract_scalar(_airbyte_data, "$['new_column']") as new_column, - json_extract_scalar(_airbyte_data, "$['date']") as date, - json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, - json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, - json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, - json_extract_scalar(_airbyte_data, "$['USD']") as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 840040e43543..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,39 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - float64 -) as id, - cast(currency as - string -) as currency, - cast(new_column as - float64 -) as new_column, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp -) as timestamp_col, - cast(HKD_special___characters as - float64 -) as HKD_special___characters, - cast(NZD as - float64 -) as NZD, - cast(USD as - float64 -) as USD, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` --- exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index 79386be9f718..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,32 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(currency as - string -), ''), '-', coalesce(cast(new_column as - string -), ''), '-', coalesce(cast(date as - string -), ''), '-', coalesce(cast(timestamp_col as - string -), ''), '-', coalesce(cast(HKD_special___characters as - string -), ''), '-', coalesce(cast(NZD as - string -), ''), '-', coalesce(cast(USD as - string -), '')) as - string -))) as _airbyte_exchange_rate_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` tmp --- exchange_rate -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 8fc7d170fb05..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` - OPTIONS() - as --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_extract_scalar(_airbyte_data, "$['id']") as id, - json_extract_scalar(_airbyte_data, "$['name']") as name, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, - json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 295be3450166..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,30 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` - OPTIONS() - as --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - int64 -) as id, - cast(name as - string -) as name, - cast(_ab_cdc_lsn as - float64 -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float64 -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float64 -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - CURRENT_TIMESTAMP() as _airbyte_normalized_at -from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` --- renamed_dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 7b882d8b693c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` as DBT_INTERNAL_DEST - using ( - select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` - ) as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd - - - - when matched then update set - `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at` - - - when not matched then insert - (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) - values - (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index b20cae81e4bf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - merge into `dataline-integration-testing`.test_normalization.`exchange_rate` as DBT_INTERNAL_DEST - using ( - select * from `dataline-integration-testing`.test_normalization.`exchange_rate__dbt_tmp` - ) as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._airbyte_ab_id = DBT_INTERNAL_DEST._airbyte_ab_id - - - - when matched then update set - `id` = DBT_INTERNAL_SOURCE.`id`,`currency` = DBT_INTERNAL_SOURCE.`currency`,`date` = DBT_INTERNAL_SOURCE.`date`,`timestamp_col` = DBT_INTERNAL_SOURCE.`timestamp_col`,`HKD_special___characters` = DBT_INTERNAL_SOURCE.`HKD_special___characters`,`NZD` = DBT_INTERNAL_SOURCE.`NZD`,`USD` = DBT_INTERNAL_SOURCE.`USD`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_exchange_rate_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_exchange_rate_hashid`,`new_column` = DBT_INTERNAL_SOURCE.`new_column` - - - when not matched then insert - (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`, `new_column`) - values - (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`, `new_column`) - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 5f8e9254f0ed..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded` as DBT_INTERNAL_DEST - using ( - select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` - ) as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key - - - - when matched then update set - `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at` - - - when not matched then insert - (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) - values - (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..2287edbeaa56 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,113 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`exchange_rate` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +with __dbt__cte__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['new_column']") as new_column, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + json_extract_scalar(_airbyte_data, "$['column___with__quotes']") as column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + float64 +) as id, + cast(currency as + string +) as currency, + cast(new_column as + float64 +) as new_column, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + cast(column___with__quotes as + string +) as column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(new_column as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), ''), '-', coalesce(cast(column___with__quotes as + string +), '')) as + string +))) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__cte__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + new_column, + date, + timestamp_col, + HKD_special___characters, + NZD, + USD, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__cte__exchange_rate_ab3 +-- exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index ad91ec4869a2..a9686f4f4e8e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -3,7 +3,61 @@ create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` OPTIONS() as --- SQL model to build a hash column based on the values of this record +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['new_column']") as new_column, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + float64 +) as id, + cast(currency as + string +) as currency, + cast(new_column as + float64 +) as new_column, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(NZD as + float64 +) as NZD, + cast(USD as + int64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select to_hex(md5(cast(concat(coalesce(cast(id as string @@ -25,7 +79,7 @@ select string ))) as _airbyte_dedup_exchange_rate_hashid, tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +from __dbt__cte__dedup_exchange_rate_ab2 tmp -- dedup_exchange_rate where 1 = 1 ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 3478ba52a70a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` - OPTIONS() - as --- SQL model to build a hash column based on the values of this record -select - to_hex(md5(cast(concat(coalesce(cast(id as - string -), ''), '-', coalesce(cast(name as - string -), ''), '-', coalesce(cast(_ab_cdc_lsn as - string -), ''), '-', coalesce(cast(_ab_cdc_updated_at as - string -), ''), '-', coalesce(cast(_ab_cdc_deleted_at as - string -), '')) as - string -))) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp --- renamed_dedup_cdc_excluded -where 1 = 1 -; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml new file mode 100755 index 000000000000..8c7494fdc58f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml @@ -0,0 +1,61 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +vars: + dbt_utils_dispatch_list: ['airbyte_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index 1567f2b78797..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_query(_airbyte_data, ''$."conflict_stream_array"'') as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index a16d74e97ea4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab1" --- conflict_stream_array -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index e5bde2f88eab..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(conflict_stream_array as - VARCHAR(max)) as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_array_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab2" tmp --- conflict_stream_array -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql deleted file mode 100644 index 711edc45508d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - json_value(conflict_stream_name, ''$."groups"'') as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql deleted file mode 100644 index a8f6a33ad157..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as - VARCHAR(max)) as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql deleted file mode 100644 index 4c510332a6ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(groups as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index 8f1fc7fe16db..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_query(_airbyte_data, ''$."conflict_stream_name"'') as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index 1e106f88c407..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab1" --- conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index dd136bf9bad9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab2" tmp --- conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index 26744bb811ae..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - json_query(conflict_stream_name, ''$."conflict_stream_name"'') as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index 6a630d1e24c1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index cbe3f02a60f7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index ecce4bc0d160..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."conflict_stream_scalar"'') as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index 0cb7a859994b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast(conflict_stream_scalar as - bigint -) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab1" --- conflict_stream_scalar -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 54f208622b20..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_scalar as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp --- conflict_stream_scalar -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql deleted file mode 100644 index d74cd1c7389f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_strea__nto_long_names_hashid, - json_query("partition", ''$."double_array_data"'') as double_array_data, - json_query("partition", ''$."DATA"'') as "DATA", - json_query("partition", ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and "partition" is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql deleted file mode 100644 index 90c0eafb817d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_''with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql deleted file mode 100644 index f898a92e06da..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(double_array_data as - VARCHAR(max)) as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("DATA" as - VARCHAR(max)) as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("column`_''with""_quotes" as - VARCHAR(max)) as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_partition_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2" tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql deleted file mode 100644 index deb088fe4931..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql +++ /dev/null @@ -1,26 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_value( - "DATA".value, ''$."currency"'') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON("DATA") - ) AS "DATA" -where 1 = 1 -and "DATA" is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql deleted file mode 100644 index 505eb01f4d31..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - VARCHAR(max)) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql deleted file mode 100644 index 89631a486695..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_partition_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_data_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2" tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql deleted file mode 100644 index 96a6f2a9eacc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql +++ /dev/null @@ -1,26 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_value( - "column`_''with""_quotes".value, ''$."currency"'') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON("column`_''with""_quotes") - ) AS "column`_''with""_quotes" -where 1 = 1 -and "column`_''with""_quotes" is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql deleted file mode 100644 index 9edfb5ade031..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - VARCHAR(max)) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql deleted file mode 100644 index 1da3ab107876..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_partition_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_column___with__quotes_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2" tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql deleted file mode 100644 index 7e83428589ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql +++ /dev/null @@ -1,26 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_value( - double_array_data.value, ''$."id"'') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON(double_array_data) - ) AS double_array_data -where 1 = 1 -and double_array_data is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql deleted file mode 100644 index 1eb3236e85d9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as - VARCHAR(max)) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql deleted file mode 100644 index b6e73981f3fb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_partition_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_double_array_data_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2" tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql deleted file mode 100644 index e921a062d3b7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."date"'') as "date", - json_query(_airbyte_data, ''$."partition"'') as "partition", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_co__lting_into_long_names -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql deleted file mode 100644 index 5eafd0d71a6b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast("date" as - VARCHAR(max)) as "date", - cast("partition" as VARCHAR(max)) as "partition", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1" --- nested_stream_with_co__lting_into_long_names -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql deleted file mode 100644 index c81d35772445..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."date"'') as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql deleted file mode 100644 index 4abb31531a75..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast("date" as - VARCHAR(max)) as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1" --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql deleted file mode 100644 index 094bd24079d4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_non_nested_s__nto_long_names_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2" tmp --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 177d8bda7d07..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_query(_airbyte_data, ''$."children"'') as children, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index a9ef564eff36..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."unnest_alias_ab1" --- unnest_alias -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index 4544841f480f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(children as - VARCHAR(max)) as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unnest_alias_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."unnest_alias_ab2" tmp --- unnest_alias -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index d9f07f78b06c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,28 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_unnest_alias_hashid, - json_value( - children.value, ''$."ab_id"'') as ab_id, - json_query( - children.value, ''$."owner"'') as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."unnest_alias" as table_alias --- children at unnest_alias/children - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON(children) - ) AS children -where 1 = 1 -and children is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index c8d4bf47083c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - bigint -) as ab_id, - cast(owner as VARCHAR(max)) as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab1" --- children at unnest_alias/children -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index 6ad3a1c4caf8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_unnest_alias_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(ab_id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(owner as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_children_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab2" tmp --- children at unnest_alias/children -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 7f49a4c7f4ea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - json_value(owner, ''$."owner_id"'') as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."unnest_alias_children" as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and owner is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 9cf0dca5efb6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - bigint -) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab1" --- owner at unnest_alias/children/owner -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index fe2cfe2e2a98..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_children_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(owner_id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_owner_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp --- owner at unnest_alias/children/owner -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql deleted file mode 100644 index a86efc7f2153..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."date"'') as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql deleted file mode 100644 index 077cbe05b163..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast("date" as - VARCHAR(max)) as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1" --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql deleted file mode 100644 index f2f9167067b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_simple_strea__nto_long_names_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2" tmp --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql index fae879eca8b1..f8d4e6017899 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -49,13 +49,13 @@ scd_data as ( "date" desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag("date") over ( + case when row_number() over ( partition by id order by "date" desc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_strea__nto_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..54e7ff118f7e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,112 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___long_names_partition_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition"','U') is not null + begin + drop table test_normalization."nested_stream_with_co___long_names_partition" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co___long_names_partition_temp_view" as + +with __dbt__CTE__nested_stream_with_co___long_names_partition_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + json_query("partition", ''$."double_array_data"'') as double_array_data, + json_query("partition", ''$."DATA"'') as "DATA", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null + +), __dbt__CTE__nested_stream_with_co___long_names_partition_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + "DATA", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co___long_names_partition_ab1 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co___long_names_partition_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(double_array_data as + VARCHAR(max)) as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("DATA" as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_partition_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co___long_names_partition_ab2 tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + "DATA", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_partition_hashid +from __dbt__CTE__nested_stream_with_co___long_names_partition_ab3 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" +where 1 = 1 + + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" FROM + "test_normalization".test_normalization."nested_stream_with_co___long_names_partition_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___long_names_partition_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___long_names_partition_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co___long_names_partition') + ) + DROP index test_normalization.nested_stream_with_co___long_names_partition.test_normalization_nested_stream_with_co___long_names_partition_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___long_names_partition_cci + ON test_normalization.nested_stream_with_co___long_names_partition + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..d94f2e859918 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,117 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___names_partition_data_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data"','U') is not null + begin + drop table test_normalization."nested_stream_with_co___names_partition_data" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co___names_partition_data_temp_view" as + +with __dbt__CTE__nested_stream_with_co___names_partition_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + "DATA".value, ''$."currency"'') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON("DATA") + ) AS "DATA" +where 1 = 1 +and "DATA" is not null + +), __dbt__CTE__nested_stream_with_co___names_partition_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + VARCHAR(max)) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co___names_partition_data_ab1 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co___names_partition_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_data_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co___names_partition_data_ab2 tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_data_hashid +from __dbt__CTE__nested_stream_with_co___names_partition_data_ab3 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___names_partition_data" FROM + "test_normalization".test_normalization."nested_stream_with_co___names_partition_data_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___names_partition_data_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___names_partition_data_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co___names_partition_data') + ) + DROP index test_normalization.nested_stream_with_co___names_partition_data.test_normalization_nested_stream_with_co___names_partition_data_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___names_partition_data_cci + ON test_normalization.nested_stream_with_co___names_partition_data + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..855eb016e760 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,117 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__ion_double_array_data_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__ion_double_array_data" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__ion_double_array_data_temp_view" as + +with __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + double_array_data.value, ''$."id"'') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON(double_array_data) + ) AS double_array_data +where 1 = 1 +and double_array_data is not null + +), __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as + VARCHAR(max)) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab1 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_double_array_data_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab2 tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab3 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data" FROM + "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__ion_double_array_data_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__ion_double_array_data_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__ion_double_array_data') + ) + DROP index test_normalization.nested_stream_with_co__ion_double_array_data.test_normalization_nested_stream_with_co__ion_double_array_data_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__ion_double_array_data_cci + ON test_normalization.nested_stream_with_co__ion_double_array_data + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index 8c21dfcb78df..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,62 +0,0 @@ - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias"','U') is not null - begin - drop table test_normalization."unnest_alias" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."unnest_alias_temp_view" as - --- Final base SQL model -select - id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from "test_normalization"._airbyte_test_normalization."unnest_alias_ab3" --- unnest_alias from "test_normalization".test_normalization._airbyte_raw_unnest_alias -where 1 = 1 - - '); - - SELECT * INTO "test_normalization".test_normalization."unnest_alias" FROM - "test_normalization".test_normalization."unnest_alias_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_unnest_alias_cci' - AND object_id=object_id('test_normalization_unnest_alias') - ) - DROP index test_normalization.unnest_alias.test_normalization_unnest_alias_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_cci - ON test_normalization.unnest_alias - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 2ffefaacaa27..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_array__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_array__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_array__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab3" --- conflict_stream_array from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_array__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_array__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_array__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_array__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_array__dbt_tmp.test_normalization_conflict_stream_array__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_array__dbt_tmp_cci - ON test_normalization.conflict_stream_array__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index e2c0ff6666d5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_name__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_name__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab3" --- conflict_stream_name from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_name__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_name__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_name__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_name__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name__dbt_tmp_cci - ON test_normalization.conflict_stream_name__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql deleted file mode 100644 index cc5f9ded53f2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - groups, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3" --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci - ON test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 79de98cb1242..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" --- conflict_stream_name at conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci - ON test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index 62ee6566a04a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_scalar__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab3" --- conflict_stream_scalar from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_scalar__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_scalar__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_scalar__dbt_tmp.test_normalization_conflict_stream_scalar__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_scalar__dbt_tmp_cci - ON test_normalization.conflict_stream_scalar__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index a1277fe7c94e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,60 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_''with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_partition_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp.test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci - ON test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index 31f9e07c8505..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_data_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp.test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci - ON test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql deleted file mode 100644 index 81ba784289fc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp.test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci - ON test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index 1e10857af549..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp.test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci - ON test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql deleted file mode 100644 index 5facac722a59..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp"','U') is not null - begin - drop table test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_non_nested_s__nto_long_names_hashid -from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3" --- non_nested_stream_wit__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" FROM - "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci' - AND object_id=object_id('test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp') - ) - DROP index test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp.test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci - ON test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 8c155fdfc51b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,59 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children__dbt_tmp"','U') is not null - begin - drop table test_normalization."unnest_alias_children__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."unnest_alias_children__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_children_hashid -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab3" --- children at unnest_alias/children from "test_normalization".test_normalization."unnest_alias" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."unnest_alias_children__dbt_tmp" FROM - "test_normalization".test_normalization."unnest_alias_children__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_unnest_alias_children__dbt_tmp_cci' - AND object_id=object_id('test_normalization_unnest_alias_children__dbt_tmp') - ) - DROP index test_normalization.unnest_alias_children__dbt_tmp.test_normalization_unnest_alias_children__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children__dbt_tmp_cci - ON test_normalization.unnest_alias_children__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index e3fbbeed2073..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp"','U') is not null - begin - drop table test_normalization."unnest_alias_children_owner__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_owner_hashid -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab3" --- owner at unnest_alias/children/owner from "test_normalization".test_normalization."unnest_alias_children" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp" FROM - "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_unnest_alias_children_owner__dbt_tmp_cci' - AND object_id=object_id('test_normalization_unnest_alias_children_owner__dbt_tmp') - ) - DROP index test_normalization.unnest_alias_children_owner__dbt_tmp.test_normalization_unnest_alias_children_owner__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children_owner__dbt_tmp_cci - ON test_normalization.unnest_alias_children_owner__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql deleted file mode 100644 index b6091947b13a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp"','U') is not null - begin - drop table test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_simple_strea__nto_long_names_hashid -from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3" --- simple_stream_with_na__lting_into_long_names from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" FROM - "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci' - AND object_id=object_id('test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp') - ) - DROP index test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp.test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci - ON test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql index 871fcf869df0..44f96a4607c9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql @@ -1,7 +1,37 @@ USE [test_normalization]; execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab3__dbt_tmp" as --- SQL model to build a hash column based on the values of this record +with __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + json_query(_airbyte_data, ''$."partition"'') as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + cast("partition" as VARCHAR(max)) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab1 +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select convert(varchar(32), HashBytes(''md5'', coalesce(cast( @@ -13,7 +43,7 @@ select VARCHAR(max)), ''''),''''), '''') as VARCHAR(max)), '''')), 2) as _airbyte_nested_strea__nto_long_names_hashid, tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2" tmp +from __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab2 tmp -- nested_stream_with_co__lting_into_long_names where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index a256ac12c37a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index e770c48bd904..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_array_ab1') }} --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index 4c6cb6138a52..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - array_to_string('conflict_stream_array'), - ]) }} as _airbyte_conflict_stream_array_hashid, - tmp.* -from {{ ref('conflict_stream_array_ab2') }} tmp --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql deleted file mode 100644 index aca9d0993391..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql deleted file mode 100644 index e7ef46dec94e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as {{ dbt_utils.type_string() }}) as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name____conflict_stream_name_ab1') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql deleted file mode 100644 index 839344888f3b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_conflict_stream_name_2_hashid', - 'groups', - ]) }} as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from {{ ref('conflict_stream_name____conflict_stream_name_ab2') }} tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index 9128769dd58b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index 7612525c0943..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_ab1') }} --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index b9d505d0445a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'conflict_stream_name', - ]) }} as _airbyte_conflict_stream_name_hashid, - tmp.* -from {{ ref('conflict_stream_name_ab2') }} tmp --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index 97ea3e982cbe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name') }} as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index 80799bfca4cf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index 1cc503d67e41..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_conflict_stream_name_hashid', - 'conflict_stream_name', - ]) }} as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index 6169df7dc10b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index 51d157d48e14..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_scalar_ab1') }} --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index f71e7c572052..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'conflict_stream_scalar', - ]) }} as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from {{ ref('conflict_stream_scalar_ab2') }} tmp --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql index a0784863bc52..76bfec9fb663 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -8,12 +8,12 @@ select _airbyte_nested_strea__nto_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, - {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co__lting_into_long_names') }} as table_alias +from {{ ref('nested_stream_with_co__lting_into_long_names_scd') }} as table_alias -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 and {{ adapter.quote('partition') }} is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql deleted file mode 100644 index aafc5126e7d6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - {{ adapter.quote('DATA') }}, - {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co___long_names_partition_ab1') }} --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql deleted file mode 100644 index a0546907dec4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_nested_strea__nto_long_names_hashid', - array_to_string('double_array_data'), - array_to_string(adapter.quote('DATA')), - array_to_string(adapter.quote('column`_\'with""_quotes')), - ]) }} as _airbyte_partition_hashid, - tmp.* -from {{ ref('nested_stream_with_co___long_names_partition_ab2') }} tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql index 46f5b7c11ae2..f4ba18cbfa77 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql @@ -1,5 +1,4 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -16,4 +15,5 @@ from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias {{ cross_join_unnest('partition', adapter.quote('DATA')) }} where 1 = 1 and {{ adapter.quote('DATA') }} is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql deleted file mode 100644 index 8c35fe50450c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co___names_partition_data_ab1') }} --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql deleted file mode 100644 index a841db507bae..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'currency', - ]) }} as _airbyte_data_hashid, - tmp.* -from {{ ref('nested_stream_with_co___names_partition_data_ab2') }} tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql deleted file mode 100644 index 8cd21390af91..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('column`_\'with""_quotes')) }} -select - _airbyte_partition_hashid, - {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} -where 1 = 1 -and {{ adapter.quote('column`_\'with""_quotes') }} is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql deleted file mode 100644 index 151455db8db0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co__column___with__quotes_ab1') }} --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql deleted file mode 100644 index fca643924c08..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'currency', - ]) }} as _airbyte_column___with__quotes_hashid, - tmp.* -from {{ ref('nested_stream_with_co__column___with__quotes_ab2') }} tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql index a191b72022c5..d194e89e0d3f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql @@ -1,5 +1,4 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -16,4 +15,5 @@ from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias {{ cross_join_unnest('partition', 'double_array_data') }} where 1 = 1 and double_array_data is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql deleted file mode 100644 index 517f1b6759ba..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co__ion_double_array_data_ab1') }} --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql deleted file mode 100644 index 5eee9d197132..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'id', - ]) }} as _airbyte_double_array_data_hashid, - tmp.* -from {{ ref('nested_stream_with_co__ion_double_array_data_ab2') }} tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql index 3b66fee00d27..7b80547c9c95 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql index fa82dea1c04a..573cd49472a8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql deleted file mode 100644 index 4214cc379780..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql deleted file mode 100644 index d68cc555b13e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('non_nested_stream_wit__lting_into_long_names_ab1') }} --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql deleted file mode 100644 index b7f3b08618eb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - adapter.quote('date'), - ]) }} as _airbyte_non_nested_s__nto_long_names_hashid, - tmp.* -from {{ ref('non_nested_stream_wit__lting_into_long_names_ab2') }} tmp --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index a7877e1d2dde..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index b397e46d7b04..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_ab1') }} --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index 9f2bd6c9d3af..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - array_to_string('children'), - ]) }} as _airbyte_unnest_alias_hashid, - tmp.* -from {{ ref('unnest_alias_ab2') }} tmp --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 7d715421a2ab..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} -select - _airbyte_unnest_alias_hashid, - {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, - {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias') }} as table_alias --- children at unnest_alias/children -{{ cross_join_unnest('unnest_alias', 'children') }} -where 1 = 1 -and children is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index cfdd1e1a68e9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, - cast(owner as {{ type_json() }}) as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children_ab1') }} --- children at unnest_alias/children -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index eb45635c9b9a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_unnest_alias_hashid', - 'ab_id', - 'owner', - ]) }} as _airbyte_children_hashid, - tmp.* -from {{ ref('unnest_alias_children_ab2') }} tmp --- children at unnest_alias/children -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 3c4f63bb121b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - {{ json_extract_scalar('owner', ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children') }} as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and owner is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 109a7652533a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children_owner_ab1') }} --- owner at unnest_alias/children/owner -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 53a716a401c1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_children_hashid', - 'owner_id', - ]) }} as _airbyte_owner_hashid, - tmp.* -from {{ ref('unnest_alias_children_owner_ab2') }} tmp --- owner at unnest_alias/children/owner -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql deleted file mode 100644 index 52601ad2201a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql deleted file mode 100644 index 28487a3d51cd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('simple_stream_with_na__lting_into_long_names_ab1') }} --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql deleted file mode 100644 index 0e264457735e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - adapter.quote('date'), - ]) }} as _airbyte_simple_strea__nto_long_names_hashid, - tmp.* -from {{ ref('simple_stream_with_na__lting_into_long_names_ab2') }} tmp --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql index b9817bc561a3..45b8e12497e0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -62,13 +62,13 @@ scd_data as ( {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( + case when row_number() over ( partition by id order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_strea__nto_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..7a3b3f978cf5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,19 @@ +{{ config( + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + {{ adapter.quote('DATA') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_partition_hashid +from {{ ref('nested_stream_with_co___long_names_partition_ab3') }} +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_co__lting_into_long_names_scd') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..cd4adb7d2c0c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,17 @@ +{{ config( + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_data_hashid +from {{ ref('nested_stream_with_co___names_partition_data_ab3') }} +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..735748fc5da3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,17 @@ +{{ config( + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from {{ ref('nested_stream_with_co__ion_double_array_data_ab3') }} +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index dad253380255..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 6e616cb068a4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from {{ ref('conflict_stream_array_ab3') }} --- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index 6b70e1601f56..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from {{ ref('conflict_stream_name_ab3') }} --- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql deleted file mode 100644 index 5244eb2cb066..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - groups, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from {{ ref('conflict_stream_name____conflict_stream_name_ab3') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index d309f67ec032..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index 3e196df09e88..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from {{ ref('conflict_stream_scalar_ab3') }} --- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index 223dd426b23f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - {{ adapter.quote('DATA') }}, - {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_partition_hashid -from {{ ref('nested_stream_with_co___long_names_partition_ab3') }} --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_co__lting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index e9f3fe55ca77..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_data_hashid -from {{ ref('nested_stream_with_co___names_partition_data_ab3') }} --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_co___long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql deleted file mode 100644 index 41147ab78466..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from {{ ref('nested_stream_with_co__column___with__quotes_ab3') }} --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_co___long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index 0ef5f8b3ad14..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from {{ ref('nested_stream_with_co__ion_double_array_data_ab3') }} --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_co___long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql deleted file mode 100644 index ac7044df4e60..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_non_nested_s__nto_long_names_hashid -from {{ ref('non_nested_stream_wit__lting_into_long_names_ab3') }} --- non_nested_stream_wit__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index aaeaad626502..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_children_hashid -from {{ ref('unnest_alias_children_ab3') }} --- children at unnest_alias/children from {{ ref('unnest_alias') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index e6710c9eb232..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_owner_hashid -from {{ ref('unnest_alias_children_owner_ab3') }} --- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql deleted file mode 100644 index a42310343b41..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization_namespace", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_simple_strea__nto_long_names_hashid -from {{ ref('simple_stream_with_na__lting_into_long_names_ab3') }} --- simple_stream_with_na__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql index 58cec3241028..0359302addb0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index 1567f2b78797..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_query(_airbyte_data, ''$."conflict_stream_array"'') as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index a16d74e97ea4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab1" --- conflict_stream_array -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index e5bde2f88eab..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(conflict_stream_array as - VARCHAR(max)) as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_array_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab2" tmp --- conflict_stream_array -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql deleted file mode 100644 index 711edc45508d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - json_value(conflict_stream_name, ''$."groups"'') as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql deleted file mode 100644 index a8f6a33ad157..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as - VARCHAR(max)) as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql deleted file mode 100644 index 4c510332a6ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(groups as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index 8f1fc7fe16db..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_query(_airbyte_data, ''$."conflict_stream_name"'') as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index 1e106f88c407..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab1" --- conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index dd136bf9bad9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab2" tmp --- conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index 26744bb811ae..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - json_query(conflict_stream_name, ''$."conflict_stream_name"'') as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index 6a630d1e24c1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index cbe3f02a60f7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index ecce4bc0d160..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."conflict_stream_scalar"'') as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index 0cb7a859994b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast(conflict_stream_scalar as - bigint -) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab1" --- conflict_stream_scalar -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 54f208622b20..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_scalar as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp --- conflict_stream_scalar -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql deleted file mode 100644 index d74cd1c7389f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_strea__nto_long_names_hashid, - json_query("partition", ''$."double_array_data"'') as double_array_data, - json_query("partition", ''$."DATA"'') as "DATA", - json_query("partition", ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and "partition" is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql deleted file mode 100644 index 90c0eafb817d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_''with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql deleted file mode 100644 index f898a92e06da..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(double_array_data as - VARCHAR(max)) as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("DATA" as - VARCHAR(max)) as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("column`_''with""_quotes" as - VARCHAR(max)) as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_partition_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2" tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql deleted file mode 100644 index deb088fe4931..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql +++ /dev/null @@ -1,26 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_value( - "DATA".value, ''$."currency"'') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON("DATA") - ) AS "DATA" -where 1 = 1 -and "DATA" is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql deleted file mode 100644 index 505eb01f4d31..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - VARCHAR(max)) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql deleted file mode 100644 index 89631a486695..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_partition_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_data_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2" tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql deleted file mode 100644 index 96a6f2a9eacc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql +++ /dev/null @@ -1,26 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_value( - "column`_''with""_quotes".value, ''$."currency"'') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON("column`_''with""_quotes") - ) AS "column`_''with""_quotes" -where 1 = 1 -and "column`_''with""_quotes" is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql deleted file mode 100644 index 9edfb5ade031..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - VARCHAR(max)) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql deleted file mode 100644 index 1da3ab107876..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_partition_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_column___with__quotes_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2" tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql deleted file mode 100644 index 7e83428589ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql +++ /dev/null @@ -1,26 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - json_value( - double_array_data.value, ''$."id"'') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON(double_array_data) - ) AS double_array_data -where 1 = 1 -and double_array_data is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql deleted file mode 100644 index 1eb3236e85d9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as - VARCHAR(max)) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql deleted file mode 100644 index b6e73981f3fb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_partition_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_double_array_data_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2" tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql deleted file mode 100644 index e921a062d3b7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."date"'') as "date", - json_query(_airbyte_data, ''$."partition"'') as "partition", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_co__lting_into_long_names -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql deleted file mode 100644 index 5eafd0d71a6b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast("date" as - VARCHAR(max)) as "date", - cast("partition" as VARCHAR(max)) as "partition", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1" --- nested_stream_with_co__lting_into_long_names -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql deleted file mode 100644 index c81d35772445..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."date"'') as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql deleted file mode 100644 index 4abb31531a75..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast("date" as - VARCHAR(max)) as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1" --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql deleted file mode 100644 index 094bd24079d4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_non_nested_s__nto_long_names_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2" tmp --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 177d8bda7d07..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_query(_airbyte_data, ''$."children"'') as children, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index a9ef564eff36..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."unnest_alias_ab1" --- unnest_alias -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index 4544841f480f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(children as - VARCHAR(max)) as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unnest_alias_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."unnest_alias_ab2" tmp --- unnest_alias -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index d9f07f78b06c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,28 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_unnest_alias_hashid, - json_value( - children.value, ''$."ab_id"'') as ab_id, - json_query( - children.value, ''$."owner"'') as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."unnest_alias" as table_alias --- children at unnest_alias/children - - CROSS APPLY ( - SELECT [value] = CASE - WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) - WHEN [type] = 5 THEN [value] - END - FROM OPENJSON(children) - ) AS children -where 1 = 1 -and children is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index c8d4bf47083c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - bigint -) as ab_id, - cast(owner as VARCHAR(max)) as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab1" --- children at unnest_alias/children -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index 6ad3a1c4caf8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_unnest_alias_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(ab_id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(owner as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_children_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab2" tmp --- children at unnest_alias/children -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 7f49a4c7f4ea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - json_value(owner, ''$."owner_id"'') as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization."unnest_alias_children" as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and owner is not null - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 9cf0dca5efb6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - bigint -) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab1" --- owner at unnest_alias/children/owner -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index fe2cfe2e2a98..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_children_hashid as - VARCHAR(max)), ''''), ''-'', coalesce(cast(owner_id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_owner_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp --- owner at unnest_alias/children/owner -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql deleted file mode 100644 index a86efc7f2153..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."date"'') as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql deleted file mode 100644 index 077cbe05b163..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - VARCHAR(max)) as id, - cast("date" as - VARCHAR(max)) as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1" --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql deleted file mode 100644 index f2f9167067b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_simple_strea__nto_long_names_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2" tmp --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..abd86939aeb0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "test_normalization".test_normalization."#nested_stream_with_co___long_names_partition__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" ("_airbyte_nested_strea__nto_long_names_hashid", "double_array_data", "DATA", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid") + ( + select "_airbyte_nested_strea__nto_long_names_hashid", "double_array_data", "DATA", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid" + from "test_normalization".test_normalization."#nested_stream_with_co___long_names_partition__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..9d52d7bbb474 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,9 @@ + + + + insert into "test_normalization".test_normalization."nested_stream_with_co___names_partition_data" ("_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid") + ( + select "_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid" + from "test_normalization".test_normalization."#nested_stream_with_co___names_partition_data__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..cc432a49e25d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,9 @@ + + + + insert into "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data" ("_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid") + ( + select "_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid" + from "test_normalization".test_normalization."#nested_stream_with_co__ion_double_array_data__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index cbef7f6209d1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "test_normalization".test_normalization."unnest_alias" - where (_airbyte_ab_id) in ( - select (_airbyte_ab_id) - from "test_normalization".test_normalization."#unnest_alias__dbt_tmp" - ); - - insert into "test_normalization".test_normalization."unnest_alias" ("id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid") - ( - select "id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid" - from "test_normalization".test_normalization."#unnest_alias__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 2ffefaacaa27..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_array__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_array__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_array__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab3" --- conflict_stream_array from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_array__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_array__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_array__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_array__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_array__dbt_tmp.test_normalization_conflict_stream_array__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_array__dbt_tmp_cci - ON test_normalization.conflict_stream_array__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index e2c0ff6666d5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_name__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_name__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab3" --- conflict_stream_name from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_name__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_name__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_name__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_name__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name__dbt_tmp_cci - ON test_normalization.conflict_stream_name__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql deleted file mode 100644 index cc5f9ded53f2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - groups, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3" --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci - ON test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 79de98cb1242..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" --- conflict_stream_name at conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci - ON test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index 62ee6566a04a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp"','U') is not null - begin - drop table test_normalization."conflict_stream_scalar__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab3" --- conflict_stream_scalar from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp" FROM - "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_conflict_stream_scalar__dbt_tmp_cci' - AND object_id=object_id('test_normalization_conflict_stream_scalar__dbt_tmp') - ) - DROP index test_normalization.conflict_stream_scalar__dbt_tmp.test_normalization_conflict_stream_scalar__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_scalar__dbt_tmp_cci - ON test_normalization.conflict_stream_scalar__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index a1277fe7c94e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,60 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_''with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_partition_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp.test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci - ON test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index 31f9e07c8505..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_data_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp.test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci - ON test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql deleted file mode 100644 index 81ba784289fc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp.test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci - ON test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index 1e10857af549..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp.test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci - ON test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql deleted file mode 100644 index 5facac722a59..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp"','U') is not null - begin - drop table test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_non_nested_s__nto_long_names_hashid -from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3" --- non_nested_stream_wit__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" FROM - "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci' - AND object_id=object_id('test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp') - ) - DROP index test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp.test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci - ON test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 8c155fdfc51b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,59 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children__dbt_tmp"','U') is not null - begin - drop table test_normalization."unnest_alias_children__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."unnest_alias_children__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_children_hashid -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab3" --- children at unnest_alias/children from "test_normalization".test_normalization."unnest_alias" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."unnest_alias_children__dbt_tmp" FROM - "test_normalization".test_normalization."unnest_alias_children__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_unnest_alias_children__dbt_tmp_cci' - AND object_id=object_id('test_normalization_unnest_alias_children__dbt_tmp') - ) - DROP index test_normalization.unnest_alias_children__dbt_tmp.test_normalization_unnest_alias_children__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children__dbt_tmp_cci - ON test_normalization.unnest_alias_children__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index e3fbbeed2073..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp"','U') is not null - begin - drop table test_normalization."unnest_alias_children_owner__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_owner_hashid -from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab3" --- owner at unnest_alias/children/owner from "test_normalization".test_normalization."unnest_alias_children" -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp" FROM - "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_unnest_alias_children_owner__dbt_tmp_cci' - AND object_id=object_id('test_normalization_unnest_alias_children_owner__dbt_tmp') - ) - DROP index test_normalization.unnest_alias_children_owner__dbt_tmp.test_normalization_unnest_alias_children_owner__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children_owner__dbt_tmp_cci - ON test_normalization.unnest_alias_children_owner__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql deleted file mode 100644 index b6091947b13a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ /dev/null @@ -1,58 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp"','U') is not null - begin - drop table test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" as - --- Final base SQL model -select - id, - "date", - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_simple_strea__nto_long_names_hashid -from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3" --- simple_stream_with_na__lting_into_long_names from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - '); - - SELECT * INTO "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" FROM - "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci' - AND object_id=object_id('test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp') - ) - DROP index test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp.test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci - ON test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql index 871fcf869df0..44f96a4607c9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql @@ -1,7 +1,37 @@ USE [test_normalization]; execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab3__dbt_tmp" as --- SQL model to build a hash column based on the values of this record +with __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + json_query(_airbyte_data, ''$."partition"'') as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + cast("partition" as VARCHAR(max)) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab1 +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select convert(varchar(32), HashBytes(''md5'', coalesce(cast( @@ -13,7 +43,7 @@ select VARCHAR(max)), ''''),''''), '''') as VARCHAR(max)), '''')), 2) as _airbyte_nested_strea__nto_long_names_hashid, tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2" tmp +from __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab2 tmp -- nested_stream_with_co__lting_into_long_names where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml new file mode 100755 index 000000000000..8c7494fdc58f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml @@ -0,0 +1,61 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +vars: + dbt_utils_dispatch_list: ['airbyte_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 7ca4f64da70a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."name"'') as name, - json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, - json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, - json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 071d2319c025..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as - VARCHAR(max)) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab1" --- dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index 62877d8eede6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."currency"'') as currency, - json_value(_airbyte_data, ''$."date"'') as "date", - json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, - json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", - json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, - json_value(_airbyte_data, ''$."NZD"'') as nzd, - json_value(_airbyte_data, ''$."USD"'') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index b871d6e41b0b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,32 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(currency as - VARCHAR(max)) as currency, - try_parse(nullif("date", '''') as date) as "date", - try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - VARCHAR(max)) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab1" --- dedup_exchange_rate -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 31eb5bb200ab..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."currency"'') as currency, - json_value(_airbyte_data, ''$."date"'') as "date", - json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, - json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", - json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, - json_value(_airbyte_data, ''$."NZD"'') as nzd, - json_value(_airbyte_data, ''$."USD"'') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index bf6e40566af3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,32 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(currency as - VARCHAR(max)) as currency, - try_parse(nullif("date", '''') as date) as "date", - try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - VARCHAR(max)) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."exchange_rate_ab1" --- exchange_rate -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index 13c53bf0f858..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,26 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as - VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as - VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as - VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as - VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."exchange_rate_ab2" tmp --- exchange_rate -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index d210ebebcdfc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."name"'') as name, - json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, - json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, - json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, - json_value(_airbyte_data, ''$."_ab_cdc_log_pos"'') as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index d23785697728..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,30 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as - VARCHAR(max)) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab1" --- pos_dedup_cdcx -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 7c05f028e95e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 362c8ff7d323..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" --- renamed_dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index b98aeb069663..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,133 +0,0 @@ - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_scd_temp_view"','V') is not null - begin - drop view test_normalization."dedup_cdc_excluded_scd_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_scd"','U') is not null - begin - drop table test_normalization."dedup_cdc_excluded_scd" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."dedup_cdc_excluded_scd_temp_view" as - -with - -input_data as ( - select * - from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab3" - -- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as - VARCHAR(max)), cast(_ab_cdc_updated_at as - VARCHAR(max)) - order by _airbyte_ab_id - ) as _airbyte_row_num, - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_unique_key as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded_scd" FROM - "test_normalization".test_normalization."dedup_cdc_excluded_scd_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_scd_temp_view"','V') is not null - begin - drop view test_normalization."dedup_cdc_excluded_scd_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded_scd_cci' - AND object_id=object_id('test_normalization_dedup_cdc_excluded_scd') - ) - DROP index test_normalization.dedup_cdc_excluded_scd.test_normalization_dedup_cdc_excluded_scd_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded_scd_cci - ON test_normalization.dedup_cdc_excluded_scd - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 60c4f8d8eab2..1a1d7fba7095 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -57,14 +57,14 @@ scd_data as ( "date" desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag("date") over ( + case when row_number() over ( partition by id, currency, cast(nzd as VARCHAR(max)) order by "date" desc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 967080a2b6ae..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,121 +0,0 @@ - - - - USE [test_normalization]; - if object_id ('test_normalization."renamed_dedup_cdc_excluded_scd_temp_view"','V') is not null - begin - drop view test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."renamed_dedup_cdc_excluded_scd"','U') is not null - begin - drop table test_normalization."renamed_dedup_cdc_excluded_scd" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" as - -with - -input_data as ( - select * - from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" - -- renamed_dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_renamed_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id - ) as _airbyte_row_num, - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_unique_key as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" FROM - "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."renamed_dedup_cdc_excluded_scd_temp_view"','V') is not null - begin - drop view test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_renamed_dedup_cdc_excluded_scd_cci' - AND object_id=object_id('test_normalization_renamed_dedup_cdc_excluded_scd') - ) - DROP index test_normalization.renamed_dedup_cdc_excluded_scd.test_normalization_renamed_dedup_cdc_excluded_scd_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_renamed_dedup_cdc_excluded_scd_cci - ON test_normalization.renamed_dedup_cdc_excluded_scd - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index c2c86219d046..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,67 +0,0 @@ - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_temp_view"','V') is not null - begin - drop view test_normalization."dedup_cdc_excluded_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded"','U') is not null - begin - drop table test_normalization."dedup_cdc_excluded" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."dedup_cdc_excluded_temp_view" as - --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from "test_normalization".test_normalization."dedup_cdc_excluded_scd" --- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded -where 1 = 1 -and _airbyte_active_row = 1 - - '); - - SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded" FROM - "test_normalization".test_normalization."dedup_cdc_excluded_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_temp_view"','V') is not null - begin - drop view test_normalization."dedup_cdc_excluded_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded_cci' - AND object_id=object_id('test_normalization_dedup_cdc_excluded') - ) - DROP index test_normalization.dedup_cdc_excluded.test_normalization_dedup_cdc_excluded_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded_cci - ON test_normalization.dedup_cdc_excluded - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 55dca5724089..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,68 +0,0 @@ - - - - USE [test_normalization]; - if object_id ('test_normalization."exchange_rate_temp_view"','V') is not null - begin - drop view test_normalization."exchange_rate_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."exchange_rate"','U') is not null - begin - drop table test_normalization."exchange_rate" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."exchange_rate_temp_view" as - --- Final base SQL model -select - id, - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from "test_normalization"._airbyte_test_normalization."exchange_rate_ab3" --- exchange_rate from "test_normalization".test_normalization._airbyte_raw_exchange_rate -where 1 = 1 - - '); - - SELECT * INTO "test_normalization".test_normalization."exchange_rate" FROM - "test_normalization".test_normalization."exchange_rate_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."exchange_rate_temp_view"','V') is not null - begin - drop view test_normalization."exchange_rate_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_exchange_rate_cci' - AND object_id=object_id('test_normalization_exchange_rate') - ) - DROP index test_normalization.exchange_rate.test_normalization_exchange_rate_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_exchange_rate_cci - ON test_normalization.exchange_rate - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 8ddf455c5ec9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,63 +0,0 @@ - - - - USE [test_normalization]; - if object_id ('test_normalization."renamed_dedup_cdc_excluded_temp_view"','V') is not null - begin - drop view test_normalization."renamed_dedup_cdc_excluded_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."renamed_dedup_cdc_excluded"','U') is not null - begin - drop table test_normalization."renamed_dedup_cdc_excluded" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."renamed_dedup_cdc_excluded_temp_view" as - --- Final base SQL model -select - _airbyte_unique_key, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" --- renamed_dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded -where 1 = 1 -and _airbyte_active_row = 1 - - '); - - SELECT * INTO "test_normalization".test_normalization."renamed_dedup_cdc_excluded" FROM - "test_normalization".test_normalization."renamed_dedup_cdc_excluded_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."renamed_dedup_cdc_excluded_temp_view"','V') is not null - begin - drop view test_normalization."renamed_dedup_cdc_excluded_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_renamed_dedup_cdc_excluded_cci' - AND object_id=object_id('test_normalization_renamed_dedup_cdc_excluded') - ) - DROP index test_normalization.renamed_dedup_cdc_excluded.test_normalization_renamed_dedup_cdc_excluded_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_renamed_dedup_cdc_excluded_cci - ON test_normalization.renamed_dedup_cdc_excluded - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 749435773423..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,134 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp"','U') is not null - begin - drop table test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" as - -with - -input_data as ( - select * - from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab3" - -- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as - VARCHAR(max)), cast(_ab_cdc_updated_at as - VARCHAR(max)), cast(_ab_cdc_log_pos as - VARCHAR(max)) - order by _airbyte_ab_id - ) as _airbyte_row_num, - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_unique_key as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" FROM - "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci' - AND object_id=object_id('test_normalization_pos_dedup_cdcx_scd__dbt_tmp') - ) - DROP index test_normalization.pos_dedup_cdcx_scd__dbt_tmp.test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci - ON test_normalization.pos_dedup_cdcx_scd__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..9b6a64b58f4e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,137 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."exchange_rate__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate__dbt_tmp"','U') is not null + begin + drop table test_normalization."exchange_rate__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."exchange_rate__dbt_tmp_temp_view" as + +with __dbt__CTE__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + json_value(_airbyte_data, ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__CTE__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse(nullif("date", '''') as date) as "date", + try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast("column`_''with""_quotes" as + VARCHAR(max)) as "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from __dbt__CTE__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__CTE__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as + VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as + VARCHAR(max)), ''''), ''-'', coalesce(cast("column`_''with""_quotes" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__CTE__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__CTE__exchange_rate_ab3 +-- exchange_rate from "test_normalization".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."exchange_rate__dbt_tmp" FROM + "test_normalization".test_normalization."exchange_rate__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."exchange_rate__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_exchange_rate__dbt_tmp_cci' + AND object_id=object_id('test_normalization_exchange_rate__dbt_tmp') + ) + DROP index test_normalization.exchange_rate__dbt_tmp.test_normalization_exchange_rate__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_exchange_rate__dbt_tmp_cci + ON test_normalization.exchange_rate__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 846de1108761..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,64 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp"','U') is not null - begin - drop table test_normalization."pos_dedup_cdcx__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from "test_normalization".test_normalization."pos_dedup_cdcx_scd" --- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and _airbyte_active_row = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp" FROM - "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx__dbt_tmp_cci' - AND object_id=object_id('test_normalization_pos_dedup_cdcx__dbt_tmp') - ) - DROP index test_normalization.pos_dedup_cdcx__dbt_tmp.test_normalization_pos_dedup_cdcx__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx__dbt_tmp_cci - ON test_normalization.pos_dedup_cdcx__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 71b4b90ce304..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,23 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(name as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp --- dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 021c2a7b88f4..e0914b27e581 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,7 +1,55 @@ USE [test_normalization]; execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as --- SQL model to build a hash column based on the values of this record +with __dbt__CTE__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__CTE__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse(nullif("date", '''') as date) as "date", + try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from __dbt__CTE__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select convert(varchar(32), HashBytes(''md5'', coalesce(cast( @@ -18,7 +66,7 @@ select VARCHAR(max)), ''''),''''), '''') as VARCHAR(max)), '''')), 2) as _airbyte_dedup_exchange_rate_hashid, tmp.* -from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp +from __dbt__CTE__dedup_exchange_rate_ab2 tmp -- dedup_exchange_rate where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index fe243c2ddc74..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,23 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(name as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp --- pos_dedup_cdcx -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 6781d88c4b0f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index bc0150c8df4e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 37994c4823f9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(name as {{ dbt_utils.type_string() }}) as name, - cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('dedup_cdc_excluded_ab1') }} --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 5ed2b660ee09..2248658f6bb0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index c20798c70b01..75e7e448eee9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index c20f382cfbc0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, - {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as {{ adapter.quote('HKD@spéçiäl & characters') }}, - {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, - {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, - {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index b737134cc6eb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - try_parse({{ empty_string_to_null(adapter.quote('date')) }} as {{ type_date() }}) as {{ adapter.quote('date') }}, - try_parse({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col, - cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }}, - cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, - cast(nzd as {{ dbt_utils.type_float() }}) as nzd, - cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('exchange_rate_ab1') }} --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index c645234a8db9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - adapter.quote('date'), - 'timestamp_col', - adapter.quote('HKD@spéçiäl & characters'), - 'hkd_special___characters', - 'nzd', - 'usd', - ]) }} as _airbyte_exchange_rate_hashid, - tmp.* -from {{ ref('exchange_rate_ab2') }} tmp --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 67bc11204a5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index a58511ceb6b7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(name as {{ dbt_utils.type_string() }}) as name, - cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('pos_dedup_cdcx_ab1') }} --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 7fd3046082a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index a04f92bdcd55..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('renamed_dedup_cdc_excluded_ab1') }} --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index d086aa78b7bc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,111 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('dedup_cdc_excluded_ab3') }} - -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('dedup_cdc_excluded_ab3') }} - -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index d406c4709941..0dce96df5afd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -71,13 +71,13 @@ scd_data as ( {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( + case when row_number() over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 67ab2bb4e05a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,103 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} - -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} - -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_renamed_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 086676f17371..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where 1 = 1 -and _airbyte_active_row = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 980126a4146c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 8a8ff85f5902..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from {{ ref('renamed_dedup_cdc_excluded_scd') }} --- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} -where 1 = 1 -and _airbyte_active_row = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 4d32358f6f28..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,113 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..73937cdf8411 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,24 @@ +{{ config( + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index fc341acc8a63..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_scd') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where 1 = 1 -and _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index dc6238af14f4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 9221c1833031..3de2443ecce3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 7dd3908bac88..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index f5ce3e6ba182..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 7ca4f64da70a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."name"'') as name, - json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, - json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, - json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 071d2319c025..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as - VARCHAR(max)) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab1" --- dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index 62877d8eede6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."currency"'') as currency, - json_value(_airbyte_data, ''$."date"'') as "date", - json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, - json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", - json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, - json_value(_airbyte_data, ''$."NZD"'') as nzd, - json_value(_airbyte_data, ''$."USD"'') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index b871d6e41b0b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,32 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(currency as - VARCHAR(max)) as currency, - try_parse(nullif("date", '''') as date) as "date", - try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - VARCHAR(max)) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab1" --- dedup_exchange_rate -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 31eb5bb200ab..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."currency"'') as currency, - json_value(_airbyte_data, ''$."date"'') as "date", - json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, - json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", - json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, - json_value(_airbyte_data, ''$."NZD"'') as nzd, - json_value(_airbyte_data, ''$."USD"'') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index bf6e40566af3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,32 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(currency as - VARCHAR(max)) as currency, - try_parse(nullif("date", '''') as date) as "date", - try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - VARCHAR(max)) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."exchange_rate_ab1" --- exchange_rate -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index 13c53bf0f858..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,26 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as - VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as - VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as - VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as - VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as - VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as - VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."exchange_rate_ab2" tmp --- exchange_rate -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index d210ebebcdfc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - json_value(_airbyte_data, ''$."name"'') as name, - json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, - json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, - json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, - json_value(_airbyte_data, ''$."_ab_cdc_log_pos"'') as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index d23785697728..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,30 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as - VARCHAR(max)) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab1" --- pos_dedup_cdcx -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 7c05f028e95e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, ''$."id"'') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 362c8ff7d323..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at -from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" --- renamed_dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index a9f7cffc2289..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "test_normalization".test_normalization."dedup_cdc_excluded_scd" - where (_airbyte_unique_key_scd) in ( - select (_airbyte_unique_key_scd) - from "test_normalization".test_normalization."#dedup_cdc_excluded_scd__dbt_tmp" - ); - - insert into "test_normalization".test_normalization."dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") - ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" - from "test_normalization".test_normalization."#dedup_cdc_excluded_scd__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 5d17dd49a167..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" - where (_airbyte_unique_key_scd) in ( - select (_airbyte_unique_key_scd) - from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded_scd__dbt_tmp" - ); - - insert into "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") - ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" - from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded_scd__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 0f76179fe523..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "test_normalization".test_normalization."dedup_cdc_excluded" - where (_airbyte_unique_key) in ( - select (_airbyte_unique_key) - from "test_normalization".test_normalization."#dedup_cdc_excluded__dbt_tmp" - ); - - insert into "test_normalization".test_normalization."dedup_cdc_excluded" ("_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") - ( - select "_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" - from "test_normalization".test_normalization."#dedup_cdc_excluded__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 993b2a0893ae..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "test_normalization".test_normalization."exchange_rate" - where (_airbyte_ab_id) in ( - select (_airbyte_ab_id) - from "test_normalization".test_normalization."#exchange_rate__dbt_tmp" - ); - - insert into "test_normalization".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") - ( - select "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid" - from "test_normalization".test_normalization."#exchange_rate__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 5b42db702fd8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "test_normalization".test_normalization."renamed_dedup_cdc_excluded" - where (_airbyte_unique_key) in ( - select (_airbyte_unique_key) - from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded__dbt_tmp" - ); - - insert into "test_normalization".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") - ( - select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" - from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 749435773423..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,134 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp"','U') is not null - begin - drop table test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" as - -with - -input_data as ( - select * - from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab3" - -- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as - VARCHAR(max)), cast(_ab_cdc_updated_at as - VARCHAR(max)), cast(_ab_cdc_log_pos as - VARCHAR(max)) - order by _airbyte_ab_id - ) as _airbyte_row_num, - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(_airbyte_unique_key as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" FROM - "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci' - AND object_id=object_id('test_normalization_pos_dedup_cdcx_scd__dbt_tmp') - ) - DROP index test_normalization.pos_dedup_cdcx_scd__dbt_tmp.test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci - ON test_normalization.pos_dedup_cdcx_scd__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..9b6a64b58f4e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,137 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."exchange_rate__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate__dbt_tmp"','U') is not null + begin + drop table test_normalization."exchange_rate__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."exchange_rate__dbt_tmp_temp_view" as + +with __dbt__CTE__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + json_value(_airbyte_data, ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__CTE__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse(nullif("date", '''') as date) as "date", + try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast("column`_''with""_quotes" as + VARCHAR(max)) as "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from __dbt__CTE__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__CTE__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as + VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as + VARCHAR(max)), ''''), ''-'', coalesce(cast("column`_''with""_quotes" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__CTE__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__CTE__exchange_rate_ab3 +-- exchange_rate from "test_normalization".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."exchange_rate__dbt_tmp" FROM + "test_normalization".test_normalization."exchange_rate__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."exchange_rate__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_exchange_rate__dbt_tmp_cci' + AND object_id=object_id('test_normalization_exchange_rate__dbt_tmp') + ) + DROP index test_normalization.exchange_rate__dbt_tmp.test_normalization_exchange_rate__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_exchange_rate__dbt_tmp_cci + ON test_normalization.exchange_rate__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 846de1108761..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,64 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp"','U') is not null - begin - drop table test_normalization."pos_dedup_cdcx__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" as - --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - SYSDATETIME() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from "test_normalization".test_normalization."pos_dedup_cdcx_scd" --- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and _airbyte_active_row = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp" FROM - "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx__dbt_tmp_cci' - AND object_id=object_id('test_normalization_pos_dedup_cdcx__dbt_tmp') - ) - DROP index test_normalization.pos_dedup_cdcx__dbt_tmp.test_normalization_pos_dedup_cdcx__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx__dbt_tmp_cci - ON test_normalization.pos_dedup_cdcx__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 71b4b90ce304..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,23 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(name as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp --- dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 021c2a7b88f4..e0914b27e581 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,7 +1,55 @@ USE [test_normalization]; execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as --- SQL model to build a hash column based on the values of this record +with __dbt__CTE__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__CTE__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse(nullif("date", '''') as date) as "date", + try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from __dbt__CTE__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select convert(varchar(32), HashBytes(''md5'', coalesce(cast( @@ -18,7 +66,7 @@ select VARCHAR(max)), ''''),''''), '''') as VARCHAR(max)), '''')), 2) as _airbyte_dedup_exchange_rate_hashid, tmp.* -from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp +from __dbt__CTE__dedup_exchange_rate_ab2 tmp -- dedup_exchange_rate where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index fe243c2ddc74..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,23 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''), ''-'', coalesce(cast(name as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as - VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp --- pos_dedup_cdcx -where 1 = 1 - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 6781d88c4b0f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as - --- SQL model to build a hash column based on the values of this record -select - convert(varchar(32), HashBytes(''md5'', coalesce(cast( - - - - concat(concat(coalesce(cast(id as - VARCHAR(max)), ''''),''''), '''') as - VARCHAR(max)), '''')), 2) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml new file mode 100755 index 000000000000..b03cee8fe930 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + # incremental is not enabled for MySql yet + #+materialized: incremental + +materialized: table + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +vars: + dbt_utils_dispatch_list: ['airbyte_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index b08ffaf4f715..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_array_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_extract(_airbyte_data, - '$."conflict_stream_array"') as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index 44b15b327edd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_array_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`conflict_stream_array_ab1` --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index 857eb62be3f6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_array_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_array as char), '')) as char)) as _airbyte_conflict_stream_array_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_array_ab2` tmp --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql deleted file mode 100644 index e3edc7461b2d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - json_extract(table_alias.conflict_stream_name, - '$."conflict_stream_name"') - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`conflict_stream_name` as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql deleted file mode 100644 index 16def64e5011..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as json) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql deleted file mode 100644 index 99164479818a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql deleted file mode 100644 index 94cef187f111..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - json_value(conflict_stream_name, - '$."groups"') as `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql deleted file mode 100644 index 222256b4cc4e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(`groups` as char) as `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql deleted file mode 100644 index 2442380dac27..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as char), ''), '-', coalesce(cast(`groups` as char), '')) as char)) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index f0cefd4a6acf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - - json_extract(table_alias._airbyte_data, - '$."conflict_stream_name"') - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index b4f1e6a94ea6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(conflict_stream_name as json) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`conflict_stream_name_ab1` --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index 8a7e62bbfafc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_name_ab2` tmp --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index a30f57d7d4ed..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_scalar_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."conflict_stream_scalar"') as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index 5822b0221381..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_scalar_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(conflict_stream_scalar as - signed -) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`conflict_stream_scalar_ab1` --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 6b31dc6f9b85..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_scalar_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_scalar as char), '')) as char)) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql deleted file mode 100644 index 2a4990a4233d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."date"') as `date`, - - json_extract(table_alias._airbyte_data, - '$."partition"') - as `partition`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_nested_s__lting_into_long_names as table_alias --- nested_stream_with_co__lting_into_long_names -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql deleted file mode 100644 index a5cd56efead8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(`date` as char) as `date`, - cast(`partition` as json) as `partition`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1` --- nested_stream_with_co__lting_into_long_names -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql deleted file mode 100644 index 328e7097da14..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_strea__nto_long_names_hashid, - json_extract(`partition`, - '$."double_array_data"') as double_array_data, - json_extract(`partition`, - '$."DATA"') as `DATA`, - json_extract(`partition`, - '$."column___with__quotes"') as `column__'with"_quotes`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`nested_stream_with_co__lting_into_long_names` as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and `partition` is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql deleted file mode 100644 index f5b912cd6800..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - `DATA`, - `column__'with"_quotes`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql deleted file mode 100644 index bf2e57812e3d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as char), ''), '-', coalesce(cast(double_array_data as char), ''), '-', coalesce(cast(`DATA` as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), '')) as char)) as _airbyte_partition_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2` tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql deleted file mode 100644 index 5dd3b6c8ce6f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql +++ /dev/null @@ -1,64 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - - ), - joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - - json_extract(double_array_data, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`nested_stream_with_co___long_names_partition` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_co___long_names_partition.double_array_data - where numbers.generated_number <= json_length(double_array_data) - ) -select - _airbyte_partition_hashid, - json_value(_airbyte_nested_data, - '$."id"') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and double_array_data is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql deleted file mode 100644 index 3280a736e4bc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as char) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql deleted file mode 100644 index f14064aa73ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(id as char), '')) as char)) as _airbyte_double_array_data_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2` tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql deleted file mode 100644 index 19ae65abddfd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql +++ /dev/null @@ -1,64 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - - ), - joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - - json_extract(`DATA`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`nested_stream_with_co___long_names_partition` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_co___long_names_partition.`DATA` - where numbers.generated_number <= json_length(`DATA`) - ) -select - _airbyte_partition_hashid, - json_value(_airbyte_nested_data, - '$."currency"') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and `DATA` is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql deleted file mode 100644 index 2a28642a67f8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as char) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql deleted file mode 100644 index 14553972ba13..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_data_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2` tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql deleted file mode 100644 index c7861eb82bd9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql +++ /dev/null @@ -1,64 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - - ), - joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - - json_extract(`column__'with"_quotes`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`nested_stream_with_co___long_names_partition` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_co___long_names_partition.`column__'with"_quotes` - where numbers.generated_number <= json_length(`column__'with"_quotes`) - ) -select - _airbyte_partition_hashid, - json_value(_airbyte_nested_data, - '$."currency"') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and `column__'with"_quotes` is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql deleted file mode 100644 index 6c71fa4683f5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as char) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql deleted file mode 100644 index b8a160636ce2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_column___with__quotes_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2` tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql deleted file mode 100644 index c428b52dd85c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."date"') as `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_non_nest__lting_into_long_names as table_alias --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql deleted file mode 100644 index 0957dfcb71cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(`date` as char) as `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1` --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql deleted file mode 100644 index 9122165fc08b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_non_nested_s__nto_long_names_hashid, - tmp.* -from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2` tmp --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 46c85dc6a760..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_extract(_airbyte_data, - '$."children"') as children, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index d0321601fc92..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`unnest_alias_ab1` --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index 9819d4b985d1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(children as char), '')) as char)) as _airbyte_unnest_alias_hashid, - tmp.* -from _airbyte_test_normalization.`unnest_alias_ab2` tmp --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index d781016d8076..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,68 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - - ), - joined as ( - select - _airbyte_unnest_alias_hashid as _airbyte_hashid, - - json_extract(children, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`unnest_alias` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in unnest_alias.children - where numbers.generated_number <= json_length(children) - ) -select - _airbyte_unnest_alias_hashid, - json_value(_airbyte_nested_data, - '$."ab_id"') as ab_id, - - json_extract(_airbyte_nested_data, - '$."owner"') - as `owner`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`unnest_alias` as table_alias --- children at unnest_alias/children -left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid -where 1 = 1 -and children is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index a2e785ac473e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - signed -) as ab_id, - cast(`owner` as json) as `owner`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`unnest_alias_children_ab1` --- children at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index a10d4db5544e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as char), ''), '-', coalesce(cast(ab_id as char), ''), '-', coalesce(cast(`owner` as char), '')) as char)) as _airbyte_children_hashid, - tmp.* -from _airbyte_test_normalization.`unnest_alias_children_ab2` tmp --- children at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 942b77e12e1d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_owner_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - json_value(`owner`, - '$."owner_id"') as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`unnest_alias_children` as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and `owner` is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 1f95376e9d66..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_owner_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - signed -) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`unnest_alias_children_owner_ab1` --- owner at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 923718fab124..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_owner_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_children_hashid as char), ''), '-', coalesce(cast(owner_id as char), '')) as char)) as _airbyte_owner_hashid, - tmp.* -from _airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp --- owner at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql deleted file mode 100644 index 72c98d266114..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."date"') as `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names as table_alias --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql deleted file mode 100644 index 3869d2131aac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(`date` as char) as `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1` --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql deleted file mode 100644 index aa7c1c724947..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_simple_strea__nto_long_names_hashid, - tmp.* -from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2` tmp --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql index bfb90b5ac404..fe6017bb2b7f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -27,13 +27,13 @@ scd_data as ( `date` desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(`date`) over ( + case when row_number() over ( partition by id order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_strea__nto_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..2c4fa84e7ced --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,67 @@ + + + create table + test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` + as ( + +with __dbt__CTE__nested_stream_with_co_2g_names_partition_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + json_extract(`partition`, + '$."double_array_data"') as double_array_data, + json_extract(`partition`, + '$."DATA"') as `DATA`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null + +), __dbt__CTE__nested_stream_with_co_2g_names_partition_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + `DATA`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab1 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co_2g_names_partition_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as char), ''), '-', coalesce(cast(double_array_data as char), ''), '-', coalesce(cast(`DATA` as char), '')) as char)) as _airbyte_partition_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab2 tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + `DATA`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_partition_hashid +from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab3 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..9af6f946cf83 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,109 @@ + + + create table + test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` + as ( + +with __dbt__CTE__nested_stream_with_co_3es_partition_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(`DATA`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.`DATA` + where numbers.generated_number <= json_length(`DATA`) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."currency"') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and `DATA` is not null + +), __dbt__CTE__nested_stream_with_co_3es_partition_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as char) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab1 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co_3es_partition_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_data_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab2 tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_data_hashid +from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab3 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..9be37bb7a61d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,109 @@ + + + create table + test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` + as ( + +with __dbt__CTE__nested_stream_with_co_3double_array_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(double_array_data, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.double_array_data + where numbers.generated_number <= json_length(double_array_data) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."id"') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null + +), __dbt__CTE__nested_stream_with_co_3double_array_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as char) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co_3double_array_data_ab1 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co_3double_array_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(id as char), '')) as char)) as _airbyte_double_array_data_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co_3double_array_data_ab2 tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from __dbt__CTE__nested_stream_with_co_3double_array_data_ab3 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index cc143bf47b05..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create table - test_normalization.`unnest_alias__dbt_tmp` - as ( - --- Final base SQL model -select - id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from _airbyte_test_normalization.`unnest_alias_ab3` --- unnest_alias from test_normalization._airbyte_raw_unnest_alias -where 1 = 1 - - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 717bc072a956..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`conflict_stream_array__dbt_tmp` - as ( - --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from _airbyte_test_normalization.`conflict_stream_array_ab3` --- conflict_stream_array from test_normalization._airbyte_raw_conflict_stream_array -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index bdfe747bf5e7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`conflict_stream_name__dbt_tmp` - as ( - --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from _airbyte_test_normalization.`conflict_stream_name_ab3` --- conflict_stream_name from test_normalization._airbyte_raw_conflict_stream_name -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql deleted file mode 100644 index ded6eda50709..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`conflict_stream_name____conflict_stream_name__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name_conflict_stream_name` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index a64da6fc2bba..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`conflict_stream_name_conflict_stream_name__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index b79e0e4c08a4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`conflict_stream_scalar__dbt_tmp` - as ( - --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from _airbyte_test_normalization.`conflict_stream_scalar_ab3` --- conflict_stream_scalar from test_normalization._airbyte_raw_conflict_stream_scalar -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index 2c30ff5645d3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - `DATA`, - `column__'with"_quotes`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_partition_hashid -from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co__lting_into_long_names` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index cc91b90e22c9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_data_hashid -from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql deleted file mode 100644 index 600adbbcf8b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co__column___with__quotes__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from test_normalization.`nested_stream_with_co___long_names_partition` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index a584e5957c67..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql deleted file mode 100644 index 9b1d5c0c792a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`non_nested_stream_wit__lting_into_long_names__dbt_tmp` - as ( - --- Final base SQL model -select - id, - `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_non_nested_s__nto_long_names_hashid -from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3` --- non_nested_stream_wit__lting_into_long_names from test_normalization._airbyte_raw_non_nest__lting_into_long_names -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 62be462c50f8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create table - test_normalization.`unnest_alias_children__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - `owner`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_children_hashid -from _airbyte_test_normalization.`unnest_alias_children_ab3` --- children at unnest_alias/children from test_normalization.`unnest_alias` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index 5747da2ce362..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`unnest_alias_children_owner__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_owner_hashid -from _airbyte_test_normalization.`unnest_alias_children_owner_ab3` --- owner at unnest_alias/children/owner from test_normalization.`unnest_alias_children` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql deleted file mode 100644 index 8a5111492bbf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization_namespace.`simple_stream_with_na__lting_into_long_names__dbt_tmp` - as ( - --- Final base SQL model -select - id, - `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_simple_strea__nto_long_names_hashid -from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3` --- simple_stream_with_na__lting_into_long_names from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql index f8c6c82e2fbc..573cea4c23a2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql @@ -1,11 +1,48 @@ create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3__dbt_tmp` as ( --- SQL model to build a hash column based on the values of this record +with __dbt__CTE__nested_stream_with_co_1g_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + + json_extract(table_alias._airbyte_data, + '$."partition"') + as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_nested_s__lting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co_1g_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + cast(`partition` as json) as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co_1g_into_long_names_ab1 +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(`partition` as char), '')) as char)) as _airbyte_nested_strea__nto_long_names_hashid, tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2` tmp +from __dbt__CTE__nested_stream_with_co_1g_into_long_names_ab2 tmp -- nested_stream_with_co__lting_into_long_names where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index a256ac12c37a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index e770c48bd904..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_array_ab1') }} --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index 4c6cb6138a52..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - array_to_string('conflict_stream_array'), - ]) }} as _airbyte_conflict_stream_array_hashid, - tmp.* -from {{ ref('conflict_stream_array_ab2') }} tmp --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql deleted file mode 100644 index 97ea3e982cbe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name') }} as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql deleted file mode 100644 index 88e3e7b5974a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name__2flict_stream_name_ab1') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql deleted file mode 100644 index b17ecef4ec82..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_conflict_stream_name_hashid', - 'conflict_stream_name', - ]) }} as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from {{ ref('conflict_stream_name__2flict_stream_name_ab2') }} tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql deleted file mode 100644 index 851bc06e78ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as {{ adapter.quote('groups') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql deleted file mode 100644 index 3bb11ee28a2e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast({{ adapter.quote('groups') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('groups') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name__3flict_stream_name_ab1') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql deleted file mode 100644 index 04a82c2387b4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_conflict_stream_name_2_hashid', - adapter.quote('groups'), - ]) }} as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from {{ ref('conflict_stream_name__3flict_stream_name_ab2') }} tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index 9128769dd58b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index 7612525c0943..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_ab1') }} --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index b9d505d0445a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'conflict_stream_name', - ]) }} as _airbyte_conflict_stream_name_hashid, - tmp.* -from {{ ref('conflict_stream_name_ab2') }} tmp --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index 6169df7dc10b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index 51d157d48e14..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_scalar_ab1') }} --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index f71e7c572052..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'conflict_stream_scalar', - ]) }} as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from {{ ref('conflict_stream_scalar_ab2') }} tmp --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql index bcca84084504..beddd4f39c6c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql index ec9afbc12ca6..7981ace17ba6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql index 686acef42df9..c1af2b22a7e6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -8,12 +8,12 @@ select _airbyte_nested_strea__nto_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, - {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column__\'with"_quotes') }}, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co__lting_into_long_names') }} as table_alias +from {{ ref('nested_stream_with_co_1g_into_long_names_scd') }} as table_alias -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 and {{ adapter.quote('partition') }} is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql deleted file mode 100644 index 17847b387ff8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - {{ adapter.quote('DATA') }}, - {{ adapter.quote('column__\'with"_quotes') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co_2g_names_partition_ab1') }} --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql deleted file mode 100644 index cd19c3cd4d2a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_nested_strea__nto_long_names_hashid', - array_to_string('double_array_data'), - array_to_string(adapter.quote('DATA')), - array_to_string(adapter.quote('column__\'with"_quotes')), - ]) }} as _airbyte_partition_hashid, - tmp.* -from {{ ref('nested_stream_with_co_2g_names_partition_ab2') }} tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql index a191b72022c5..d194e89e0d3f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql @@ -1,5 +1,4 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -16,4 +15,5 @@ from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias {{ cross_join_unnest('partition', 'double_array_data') }} where 1 = 1 and double_array_data is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql deleted file mode 100644 index 685ebd8cf4f0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co_3double_array_data_ab1') }} --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql deleted file mode 100644 index d3d654a0b267..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'id', - ]) }} as _airbyte_double_array_data_hashid, - tmp.* -from {{ ref('nested_stream_with_co_3double_array_data_ab2') }} tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql index 46f5b7c11ae2..f4ba18cbfa77 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql @@ -1,5 +1,4 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -16,4 +15,5 @@ from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias {{ cross_join_unnest('partition', adapter.quote('DATA')) }} where 1 = 1 and {{ adapter.quote('DATA') }} is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql deleted file mode 100644 index b6912038bca1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co_3es_partition_data_ab1') }} --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql deleted file mode 100644 index cc049c42b2c4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'currency', - ]) }} as _airbyte_data_hashid, - tmp.* -from {{ ref('nested_stream_with_co_3es_partition_data_ab2') }} tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql deleted file mode 100644 index 8b23b31f54b9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('column__\'with"_quotes')) }} -select - _airbyte_partition_hashid, - {{ json_extract_scalar(unnested_column_value(adapter.quote('column__\'with"_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -{{ cross_join_unnest('partition', adapter.quote('column__\'with"_quotes')) }} -where 1 = 1 -and {{ adapter.quote('column__\'with"_quotes') }} is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql deleted file mode 100644 index 4596c0998414..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co_3mn___with__quotes_ab1') }} --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql deleted file mode 100644 index 640685bc8c6a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'currency', - ]) }} as _airbyte_column___with__quotes_hashid, - tmp.* -from {{ ref('nested_stream_with_co_3mn___with__quotes_ab2') }} tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql deleted file mode 100644 index 4e89cc3ec03c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_non_nest__lting_into_long_names') }} as table_alias --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql deleted file mode 100644 index c1bc0cb6750f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('non_nested_stream_wit_1g_into_long_names_ab1') }} --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql deleted file mode 100644 index 81d16c7714e9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - adapter.quote('date'), - ]) }} as _airbyte_non_nested_s__nto_long_names_hashid, - tmp.* -from {{ ref('non_nested_stream_wit_1g_into_long_names_ab2') }} tmp --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index a7877e1d2dde..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index b397e46d7b04..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_ab1') }} --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index 9f2bd6c9d3af..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - array_to_string('children'), - ]) }} as _airbyte_unnest_alias_hashid, - tmp.* -from {{ ref('unnest_alias_ab2') }} tmp --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 0ae376bf5c5e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} -select - _airbyte_unnest_alias_hashid, - {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, - {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as {{ adapter.quote('owner') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias') }} as table_alias --- children at unnest_alias/children -{{ cross_join_unnest('unnest_alias', 'children') }} -where 1 = 1 -and children is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index 4e285cc11ccc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, - cast({{ adapter.quote('owner') }} as {{ type_json() }}) as {{ adapter.quote('owner') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children_ab1') }} --- children at unnest_alias/children -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index 71b013d95e95..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_unnest_alias_hashid', - 'ab_id', - adapter.quote('owner'), - ]) }} as _airbyte_children_hashid, - tmp.* -from {{ ref('unnest_alias_children_ab2') }} tmp --- children at unnest_alias/children -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 833b7054705a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - {{ json_extract_scalar(adapter.quote('owner'), ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children') }} as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and {{ adapter.quote('owner') }} is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 109a7652533a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children_owner_ab1') }} --- owner at unnest_alias/children/owner -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 53a716a401c1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_children_hashid', - 'owner_id', - ]) }} as _airbyte_owner_hashid, - tmp.* -from {{ ref('unnest_alias_children_owner_ab2') }} tmp --- owner at unnest_alias/children/owner -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql deleted file mode 100644 index 15a7e0bc2774..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization_namespace', '_airbyte_raw_simple_s__lting_into_long_names') }} as table_alias --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql deleted file mode 100644 index b37432da5333..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('simple_stream_with_na_1g_into_long_names_ab1') }} --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql deleted file mode 100644 index bfe41baca551..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - adapter.quote('date'), - ]) }} as _airbyte_simple_strea__nto_long_names_hashid, - tmp.* -from {{ ref('simple_stream_with_na_1g_into_long_names_ab2') }} tmp --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql index 98783c3a173d..42c5fa89608f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -62,13 +62,13 @@ scd_data as ( {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( + case when row_number() over ( partition by id order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_strea__nto_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..6dc787093a63 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,19 @@ +{{ config( + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + {{ adapter.quote('DATA') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_partition_hashid +from {{ ref('nested_stream_with_co_2g_names_partition_ab3') }} +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_co_1g_into_long_names_scd') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..c85eb65cf373 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,17 @@ +{{ config( + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_data_hashid +from {{ ref('nested_stream_with_co_3es_partition_data_ab3') }} +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..58b5db38588a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,17 @@ +{{ config( + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from {{ ref('nested_stream_with_co_3double_array_data_ab3') }} +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index dad253380255..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 6e616cb068a4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from {{ ref('conflict_stream_array_ab3') }} --- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index 6b70e1601f56..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from {{ ref('conflict_stream_name_ab3') }} --- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql deleted file mode 100644 index 81f173664209..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - {{ adapter.quote('groups') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from {{ ref('conflict_stream_name__3flict_stream_name_ab3') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index be1b4fe446a4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from {{ ref('conflict_stream_name__2flict_stream_name_ab3') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index 3e196df09e88..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from {{ ref('conflict_stream_scalar_ab3') }} --- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index f5f0f16333f8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - {{ adapter.quote('DATA') }}, - {{ adapter.quote('column__\'with"_quotes') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_partition_hashid -from {{ ref('nested_stream_with_co_2g_names_partition_ab3') }} --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_co__lting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index 5219d4bf1eb9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_data_hashid -from {{ ref('nested_stream_with_co_3es_partition_data_ab3') }} --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_co___long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql deleted file mode 100644 index b56e3f5b5da3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from {{ ref('nested_stream_with_co_3mn___with__quotes_ab3') }} --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_co___long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index 1cd2763acbac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from {{ ref('nested_stream_with_co_3double_array_data_ab3') }} --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_co___long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql deleted file mode 100644 index 80f9bdb879a3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_non_nested_s__nto_long_names_hashid -from {{ ref('non_nested_stream_wit_1g_into_long_names_ab3') }} --- non_nested_stream_wit__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nest__lting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 59c7f1b6c50d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - {{ adapter.quote('owner') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_children_hashid -from {{ ref('unnest_alias_children_ab3') }} --- children at unnest_alias/children from {{ ref('unnest_alias') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index e6710c9eb232..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_owner_hashid -from {{ ref('unnest_alias_children_owner_ab3') }} --- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql deleted file mode 100644 index da684535a739..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization_namespace", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_simple_strea__nto_long_names_hashid -from {{ ref('simple_stream_with_na_1g_into_long_names_ab3') }} --- simple_stream_with_na__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_s__lting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql index 1e93090d9c40..cb51bc0f04b1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index b08ffaf4f715..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_array_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_extract(_airbyte_data, - '$."conflict_stream_array"') as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index 44b15b327edd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_array_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`conflict_stream_array_ab1` --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index 857eb62be3f6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_array_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_array as char), '')) as char)) as _airbyte_conflict_stream_array_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_array_ab2` tmp --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql deleted file mode 100644 index e3edc7461b2d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - json_extract(table_alias.conflict_stream_name, - '$."conflict_stream_name"') - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`conflict_stream_name` as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql deleted file mode 100644 index 16def64e5011..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as json) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql deleted file mode 100644 index 99164479818a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql deleted file mode 100644 index 94cef187f111..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - json_value(conflict_stream_name, - '$."groups"') as `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql deleted file mode 100644 index 222256b4cc4e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(`groups` as char) as `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql deleted file mode 100644 index 2442380dac27..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as char), ''), '-', coalesce(cast(`groups` as char), '')) as char)) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index f0cefd4a6acf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - - json_extract(table_alias._airbyte_data, - '$."conflict_stream_name"') - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index b4f1e6a94ea6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(conflict_stream_name as json) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`conflict_stream_name_ab1` --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index 8a7e62bbfafc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_name_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_name_ab2` tmp --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index a30f57d7d4ed..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_scalar_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."conflict_stream_scalar"') as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index 5822b0221381..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_scalar_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(conflict_stream_scalar as - signed -) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`conflict_stream_scalar_ab1` --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 6b31dc6f9b85..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`conflict_stream_scalar_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_scalar as char), '')) as char)) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from _airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql deleted file mode 100644 index 2a4990a4233d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."date"') as `date`, - - json_extract(table_alias._airbyte_data, - '$."partition"') - as `partition`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_nested_s__lting_into_long_names as table_alias --- nested_stream_with_co__lting_into_long_names -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql deleted file mode 100644 index a5cd56efead8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(`date` as char) as `date`, - cast(`partition` as json) as `partition`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1` --- nested_stream_with_co__lting_into_long_names -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql deleted file mode 100644 index 328e7097da14..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql +++ /dev/null @@ -1,22 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_strea__nto_long_names_hashid, - json_extract(`partition`, - '$."double_array_data"') as double_array_data, - json_extract(`partition`, - '$."DATA"') as `DATA`, - json_extract(`partition`, - '$."column___with__quotes"') as `column__'with"_quotes`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`nested_stream_with_co__lting_into_long_names` as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and `partition` is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql deleted file mode 100644 index f5b912cd6800..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - `DATA`, - `column__'with"_quotes`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql deleted file mode 100644 index bf2e57812e3d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as char), ''), '-', coalesce(cast(double_array_data as char), ''), '-', coalesce(cast(`DATA` as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), '')) as char)) as _airbyte_partition_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2` tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql deleted file mode 100644 index 5dd3b6c8ce6f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql +++ /dev/null @@ -1,64 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - - ), - joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - - json_extract(double_array_data, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`nested_stream_with_co___long_names_partition` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_co___long_names_partition.double_array_data - where numbers.generated_number <= json_length(double_array_data) - ) -select - _airbyte_partition_hashid, - json_value(_airbyte_nested_data, - '$."id"') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and double_array_data is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql deleted file mode 100644 index 3280a736e4bc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as char) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql deleted file mode 100644 index f14064aa73ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(id as char), '')) as char)) as _airbyte_double_array_data_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2` tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql deleted file mode 100644 index 19ae65abddfd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql +++ /dev/null @@ -1,64 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - - ), - joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - - json_extract(`DATA`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`nested_stream_with_co___long_names_partition` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_co___long_names_partition.`DATA` - where numbers.generated_number <= json_length(`DATA`) - ) -select - _airbyte_partition_hashid, - json_value(_airbyte_nested_data, - '$."currency"') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and `DATA` is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql deleted file mode 100644 index 2a28642a67f8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as char) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql deleted file mode 100644 index 14553972ba13..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_data_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2` tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql deleted file mode 100644 index c7861eb82bd9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql +++ /dev/null @@ -1,64 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - - ), - joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - - json_extract(`column__'with"_quotes`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`nested_stream_with_co___long_names_partition` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_co___long_names_partition.`column__'with"_quotes` - where numbers.generated_number <= json_length(`column__'with"_quotes`) - ) -select - _airbyte_partition_hashid, - json_value(_airbyte_nested_data, - '$."currency"') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and `column__'with"_quotes` is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql deleted file mode 100644 index 6c71fa4683f5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as char) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql deleted file mode 100644 index b8a160636ce2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_column___with__quotes_hashid, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2` tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql deleted file mode 100644 index c428b52dd85c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."date"') as `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_non_nest__lting_into_long_names as table_alias --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql deleted file mode 100644 index 0957dfcb71cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(`date` as char) as `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1` --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql deleted file mode 100644 index 9122165fc08b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_non_nested_s__nto_long_names_hashid, - tmp.* -from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2` tmp --- non_nested_stream_wit__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index 46c85dc6a760..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_extract(_airbyte_data, - '$."children"') as children, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index d0321601fc92..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`unnest_alias_ab1` --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index 9819d4b985d1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(children as char), '')) as char)) as _airbyte_unnest_alias_hashid, - tmp.* -from _airbyte_test_normalization.`unnest_alias_ab2` tmp --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index d781016d8076..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,68 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * pow(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - - ), - joined as ( - select - _airbyte_unnest_alias_hashid as _airbyte_hashid, - - json_extract(children, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data - from test_normalization.`unnest_alias` - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in unnest_alias.children - where numbers.generated_number <= json_length(children) - ) -select - _airbyte_unnest_alias_hashid, - json_value(_airbyte_nested_data, - '$."ab_id"') as ab_id, - - json_extract(_airbyte_nested_data, - '$."owner"') - as `owner`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`unnest_alias` as table_alias --- children at unnest_alias/children -left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid -where 1 = 1 -and children is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index a2e785ac473e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - signed -) as ab_id, - cast(`owner` as json) as `owner`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`unnest_alias_children_ab1` --- children at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index a10d4db5544e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as char), ''), '-', coalesce(cast(ab_id as char), ''), '-', coalesce(cast(`owner` as char), '')) as char)) as _airbyte_children_hashid, - tmp.* -from _airbyte_test_normalization.`unnest_alias_children_ab2` tmp --- children at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 942b77e12e1d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_owner_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - json_value(`owner`, - '$."owner_id"') as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization.`unnest_alias_children` as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and `owner` is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index 1f95376e9d66..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_owner_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - signed -) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`unnest_alias_children_owner_ab1` --- owner at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 923718fab124..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`unnest_alias_children_owner_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(_airbyte_children_hashid as char), ''), '-', coalesce(cast(owner_id as char), '')) as char)) as _airbyte_owner_hashid, - tmp.* -from _airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp --- owner at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql deleted file mode 100644 index 72c98d266114..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."date"') as `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names as table_alias --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql deleted file mode 100644 index 3869d2131aac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as char) as id, - cast(`date` as char) as `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1` --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql deleted file mode 100644 index aa7c1c724947..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_simple_strea__nto_long_names_hashid, - tmp.* -from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2` tmp --- simple_stream_with_na__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql index bfb90b5ac404..fe6017bb2b7f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -27,13 +27,13 @@ scd_data as ( `date` desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(`date`) over ( + case when row_number() over ( partition by id order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_strea__nto_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..2c4fa84e7ced --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,67 @@ + + + create table + test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` + as ( + +with __dbt__CTE__nested_stream_with_co_2g_names_partition_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + json_extract(`partition`, + '$."double_array_data"') as double_array_data, + json_extract(`partition`, + '$."DATA"') as `DATA`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null + +), __dbt__CTE__nested_stream_with_co_2g_names_partition_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + `DATA`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab1 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co_2g_names_partition_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as char), ''), '-', coalesce(cast(double_array_data as char), ''), '-', coalesce(cast(`DATA` as char), '')) as char)) as _airbyte_partition_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab2 tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + `DATA`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_partition_hashid +from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab3 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..9af6f946cf83 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,109 @@ + + + create table + test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` + as ( + +with __dbt__CTE__nested_stream_with_co_3es_partition_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(`DATA`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.`DATA` + where numbers.generated_number <= json_length(`DATA`) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."currency"') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and `DATA` is not null + +), __dbt__CTE__nested_stream_with_co_3es_partition_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as char) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab1 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co_3es_partition_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_data_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab2 tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_data_hashid +from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab3 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..9be37bb7a61d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,109 @@ + + + create table + test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` + as ( + +with __dbt__CTE__nested_stream_with_co_3double_array_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(double_array_data, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.double_array_data + where numbers.generated_number <= json_length(double_array_data) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."id"') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null + +), __dbt__CTE__nested_stream_with_co_3double_array_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as char) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co_3double_array_data_ab1 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co_3double_array_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(id as char), '')) as char)) as _airbyte_double_array_data_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co_3double_array_data_ab2 tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from __dbt__CTE__nested_stream_with_co_3double_array_data_ab3 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index cc143bf47b05..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create table - test_normalization.`unnest_alias__dbt_tmp` - as ( - --- Final base SQL model -select - id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from _airbyte_test_normalization.`unnest_alias_ab3` --- unnest_alias from test_normalization._airbyte_raw_unnest_alias -where 1 = 1 - - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 717bc072a956..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`conflict_stream_array__dbt_tmp` - as ( - --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from _airbyte_test_normalization.`conflict_stream_array_ab3` --- conflict_stream_array from test_normalization._airbyte_raw_conflict_stream_array -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index bdfe747bf5e7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`conflict_stream_name__dbt_tmp` - as ( - --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from _airbyte_test_normalization.`conflict_stream_name_ab3` --- conflict_stream_name from test_normalization._airbyte_raw_conflict_stream_name -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql deleted file mode 100644 index ded6eda50709..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`conflict_stream_name____conflict_stream_name__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - `groups`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name_conflict_stream_name` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index a64da6fc2bba..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`conflict_stream_name_conflict_stream_name__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3` --- conflict_stream_name at conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index b79e0e4c08a4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`conflict_stream_scalar__dbt_tmp` - as ( - --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from _airbyte_test_normalization.`conflict_stream_scalar_ab3` --- conflict_stream_scalar from test_normalization._airbyte_raw_conflict_stream_scalar -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index 2c30ff5645d3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - `DATA`, - `column__'with"_quotes`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_partition_hashid -from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co__lting_into_long_names` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index cc91b90e22c9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_data_hashid -from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql deleted file mode 100644 index 600adbbcf8b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co__column___with__quotes__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from test_normalization.`nested_stream_with_co___long_names_partition` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index a584e5957c67..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql deleted file mode 100644 index 9b1d5c0c792a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`non_nested_stream_wit__lting_into_long_names__dbt_tmp` - as ( - --- Final base SQL model -select - id, - `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_non_nested_s__nto_long_names_hashid -from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3` --- non_nested_stream_wit__lting_into_long_names from test_normalization._airbyte_raw_non_nest__lting_into_long_names -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 62be462c50f8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create table - test_normalization.`unnest_alias_children__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - `owner`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_children_hashid -from _airbyte_test_normalization.`unnest_alias_children_ab3` --- children at unnest_alias/children from test_normalization.`unnest_alias` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index 5747da2ce362..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization.`unnest_alias_children_owner__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_owner_hashid -from _airbyte_test_normalization.`unnest_alias_children_owner_ab3` --- owner at unnest_alias/children/owner from test_normalization.`unnest_alias_children` -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql deleted file mode 100644 index 8a5111492bbf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table - test_normalization_namespace.`simple_stream_with_na__lting_into_long_names__dbt_tmp` - as ( - --- Final base SQL model -select - id, - `date`, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_simple_strea__nto_long_names_hashid -from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3` --- simple_stream_with_na__lting_into_long_names from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names -where 1 = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql index f8c6c82e2fbc..573cea4c23a2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql @@ -1,11 +1,48 @@ create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3__dbt_tmp` as ( --- SQL model to build a hash column based on the values of this record +with __dbt__CTE__nested_stream_with_co_1g_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + + json_extract(table_alias._airbyte_data, + '$."partition"') + as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_nested_s__lting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + +), __dbt__CTE__nested_stream_with_co_1g_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + cast(`partition` as json) as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__nested_stream_with_co_1g_into_long_names_ab1 +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(`partition` as char), '')) as char)) as _airbyte_nested_strea__nto_long_names_hashid, tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2` tmp +from __dbt__CTE__nested_stream_with_co_1g_into_long_names_ab2 tmp -- nested_stream_with_co__lting_into_long_names where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml new file mode 100755 index 000000000000..b03cee8fe930 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + # incremental is not enabled for MySql yet + #+materialized: incremental + +materialized: table + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +vars: + dbt_utils_dispatch_list: ['airbyte_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 33bc1c9a474e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,25 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."name"') as `name`, - json_value(_airbyte_data, - '$."_ab_cdc_lsn"') as _ab_cdc_lsn, - json_value(_airbyte_data, - '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, - json_value(_airbyte_data, - '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index c725452b87b1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(`name` as char) as `name`, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`dedup_cdc_excluded_ab1` --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index e42ac40db163..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,31 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."currency"') as currency, - json_value(_airbyte_data, - '$."date"') as `date`, - json_value(_airbyte_data, - '$."timestamp_col"') as timestamp_col, - json_value(_airbyte_data, - '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, - json_value(_airbyte_data, - '$."HKD_special___characters"') as hkd_special___characters, - json_value(_airbyte_data, - '$."NZD"') as nzd, - json_value(_airbyte_data, - '$."USD"') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index 489196399610..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,34 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(currency as char) as currency, - case when `date` = '' then NULL - else cast(`date` as date) - end as `date` - , - cast(nullif(timestamp_col, '') as char) as timestamp_col, - cast(`HKD@spéçiäl & characters` as - float -) as `HKD@spéçiäl & characters`, - cast(hkd_special___characters as char) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`dedup_exchange_rate_ab1` --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 335aa00cf89f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,31 +0,0 @@ - - create view _airbyte_test_normalization.`exchange_rate_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."currency"') as currency, - json_value(_airbyte_data, - '$."date"') as `date`, - json_value(_airbyte_data, - '$."timestamp_col"') as timestamp_col, - json_value(_airbyte_data, - '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, - json_value(_airbyte_data, - '$."HKD_special___characters"') as hkd_special___characters, - json_value(_airbyte_data, - '$."NZD"') as nzd, - json_value(_airbyte_data, - '$."USD"') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 258e3f4caf08..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,34 +0,0 @@ - - create view _airbyte_test_normalization.`exchange_rate_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(currency as char) as currency, - case when `date` = '' then NULL - else cast(`date` as date) - end as `date` - , - cast(nullif(timestamp_col, '') as char) as timestamp_col, - cast(`HKD@spéçiäl & characters` as - float -) as `HKD@spéçiäl & characters`, - cast(hkd_special___characters as char) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`exchange_rate_ab1` --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index 6cc3a26b097f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view _airbyte_test_normalization.`exchange_rate_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_exchange_rate_hashid, - tmp.* -from _airbyte_test_normalization.`exchange_rate_ab2` tmp --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 7fd41343ac43..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,26 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."name"') as `name`, - json_value(_airbyte_data, - '$."_ab_cdc_lsn"') as _ab_cdc_lsn, - json_value(_airbyte_data, - '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, - json_value(_airbyte_data, - '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, - json_value(_airbyte_data, - '$."_ab_cdc_log_pos"') as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index 9f8c55f05dcf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(`name` as char) as `name`, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`pos_dedup_cdcx_ab1` --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 446b1c167a26..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 01fe01259f91..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 4dd4da0f3431..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,74 +0,0 @@ - - - create table - test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` - as ( - -with - -input_data as ( - select * - from _airbyte_test_normalization.`dedup_cdc_excluded_ab3` - -- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char) - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), '')) as char)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 03eb062ccd38..6015fa2a1960 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -32,13 +32,13 @@ scd_data as ( `date` desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(`date`) over ( + case when row_number() over ( partition by id, currency, cast(nzd as char) order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 13bbe0a227a9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,66 +0,0 @@ - - - create table - test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` - as ( - -with - -input_data as ( - select * - from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` - -- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_renamed_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 37503d5bb636..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create table - test_normalization.`dedup_cdc_excluded__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from test_normalization.`dedup_cdc_excluded_scd` --- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded -where 1 = 1 -and _airbyte_active_row = 1 - - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 6b8a3751184f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - create table - test_normalization.`exchange_rate__dbt_tmp` - as ( - --- Final base SQL model -select - id, - currency, - `date`, - timestamp_col, - `HKD@spéçiäl & characters`, - hkd_special___characters, - nzd, - usd, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from _airbyte_test_normalization.`exchange_rate_ab3` --- exchange_rate from test_normalization._airbyte_raw_exchange_rate -where 1 = 1 - - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 10a547a0cccb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create table - test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from test_normalization.`renamed_dedup_cdc_excluded_scd` --- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded -where 1 = 1 -and _airbyte_active_row = 1 - - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index be6a9a952a26..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,76 +0,0 @@ - - - create table - test_normalization.`pos_dedup_cdcx_scd__dbt_tmp` - as ( - -with - -input_data as ( - select * - from _airbyte_test_normalization.`pos_dedup_cdcx_ab3` - -- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char), cast(_ab_cdc_log_pos as char) - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..f915bc6aceb6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,98 @@ + + + create table + test_normalization.`exchange_rate__dbt_tmp` + as ( + +with __dbt__CTE__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."currency"') as currency, + json_value(_airbyte_data, + '$."date"') as `date`, + json_value(_airbyte_data, + '$."timestamp_col"') as timestamp_col, + json_value(_airbyte_data, + '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + json_value(_airbyte_data, + '$."HKD_special___characters"') as hkd_special___characters, + json_value(_airbyte_data, + '$."NZD"') as nzd, + json_value(_airbyte_data, + '$."USD"') as usd, + json_value(_airbyte_data, + '$."column___with__quotes"') as `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__CTE__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(currency as char) as currency, + case when `date` = '' then NULL + else cast(`date` as date) + end as `date` + , + cast(nullif(timestamp_col, '') as char) as timestamp_col, + cast(`HKD@spéçiäl & characters` as + float +) as `HKD@spéçiäl & characters`, + cast(hkd_special___characters as char) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast(`column__'with"_quotes` as char) as `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__CTE__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), '')) as char)) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__CTE__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__CTE__exchange_rate_ab3 +-- exchange_rate from test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 736d35c45a9a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create table - test_normalization.`pos_dedup_cdcx__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from test_normalization.`pos_dedup_cdcx_scd` --- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and _airbyte_active_row = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 3caccb347235..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), '')) as char)) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from _airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 98129db90185..12e72758140b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,11 +1,72 @@ create view _airbyte_test_normalization.`dedup_exchange_rate_ab3__dbt_tmp` as ( --- SQL model to build a hash column based on the values of this record +with __dbt__CTE__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."currency"') as currency, + json_value(_airbyte_data, + '$."date"') as `date`, + json_value(_airbyte_data, + '$."timestamp_col"') as timestamp_col, + json_value(_airbyte_data, + '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + json_value(_airbyte_data, + '$."HKD_special___characters"') as hkd_special___characters, + json_value(_airbyte_data, + '$."NZD"') as nzd, + json_value(_airbyte_data, + '$."USD"') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__CTE__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(currency as char) as currency, + case when `date` = '' then NULL + else cast(`date` as date) + end as `date` + , + cast(nullif(timestamp_col, '') as char) as timestamp_col, + cast(`HKD@spéçiäl & characters` as + float +) as `HKD@spéçiäl & characters`, + cast(hkd_special___characters as char) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_dedup_exchange_rate_hashid, tmp.* -from _airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +from __dbt__CTE__dedup_exchange_rate_ab2 tmp -- dedup_exchange_rate where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 857c87e6fc1d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from _airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 2df6fcdc3715..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 5fb29052e85b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as {{ adapter.quote('name') }}, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 42bbbbc98de8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast({{ adapter.quote('name') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('name') }}, - cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('dedup_cdc_excluded_ab1') }} --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 5ed2b660ee09..2248658f6bb0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index a45b5767c899..c375a05ffb89 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index c20f382cfbc0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, - {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as {{ adapter.quote('HKD@spéçiäl & characters') }}, - {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, - {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, - {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 733803bde010..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - case when {{ adapter.quote('date') }} = '' then NULL - else cast({{ adapter.quote('date') }} as date) - end as {{ adapter.quote('date') }} - , - cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col, - cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }}, - cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, - cast(nzd as {{ dbt_utils.type_float() }}) as nzd, - cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('exchange_rate_ab1') }} --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index c645234a8db9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - adapter.quote('date'), - 'timestamp_col', - adapter.quote('HKD@spéçiäl & characters'), - 'hkd_special___characters', - 'nzd', - 'usd', - ]) }} as _airbyte_exchange_rate_hashid, - tmp.* -from {{ ref('exchange_rate_ab2') }} tmp --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index ac502fab3a57..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as {{ adapter.quote('name') }}, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index f6664f48d0bb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast({{ adapter.quote('name') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('name') }}, - cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('pos_dedup_cdcx_ab1') }} --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 7fd3046082a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index a04f92bdcd55..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('renamed_dedup_cdc_excluded_ab1') }} --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index ff4320b54f64..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,111 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('dedup_cdc_excluded_ab3') }} - -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('dedup_cdc_excluded_ab3') }} - -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 470401fb5740..80540be7a7a6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -71,13 +71,13 @@ scd_data as ( {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( + case when row_number() over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index b4ae7c90fe73..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,103 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} - -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} - -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_renamed_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index af2a08ea048f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where 1 = 1 -and _airbyte_active_row = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 980126a4146c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 8a8ff85f5902..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from {{ ref('renamed_dedup_cdc_excluded_scd') }} --- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} -where 1 = 1 -and _airbyte_active_row = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 7a3a501bb98f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,113 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..a74e49bd9ad7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,24 @@ +{{ config( + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('column__\'with"_quotes') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 1a756d5b7507..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_scd') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where 1 = 1 -and _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 6d34fc802057..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - adapter.quote('name'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 9221c1833031..3de2443ecce3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 78def1d5d1aa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - adapter.quote('name'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index f5ce3e6ba182..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 33bc1c9a474e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,25 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."name"') as `name`, - json_value(_airbyte_data, - '$."_ab_cdc_lsn"') as _ab_cdc_lsn, - json_value(_airbyte_data, - '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, - json_value(_airbyte_data, - '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index c725452b87b1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(`name` as char) as `name`, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`dedup_cdc_excluded_ab1` --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index e42ac40db163..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,31 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."currency"') as currency, - json_value(_airbyte_data, - '$."date"') as `date`, - json_value(_airbyte_data, - '$."timestamp_col"') as timestamp_col, - json_value(_airbyte_data, - '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, - json_value(_airbyte_data, - '$."HKD_special___characters"') as hkd_special___characters, - json_value(_airbyte_data, - '$."NZD"') as nzd, - json_value(_airbyte_data, - '$."USD"') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index 489196399610..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,34 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(currency as char) as currency, - case when `date` = '' then NULL - else cast(`date` as date) - end as `date` - , - cast(nullif(timestamp_col, '') as char) as timestamp_col, - cast(`HKD@spéçiäl & characters` as - float -) as `HKD@spéçiäl & characters`, - cast(hkd_special___characters as char) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`dedup_exchange_rate_ab1` --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 335aa00cf89f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,31 +0,0 @@ - - create view _airbyte_test_normalization.`exchange_rate_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."currency"') as currency, - json_value(_airbyte_data, - '$."date"') as `date`, - json_value(_airbyte_data, - '$."timestamp_col"') as timestamp_col, - json_value(_airbyte_data, - '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, - json_value(_airbyte_data, - '$."HKD_special___characters"') as hkd_special___characters, - json_value(_airbyte_data, - '$."NZD"') as nzd, - json_value(_airbyte_data, - '$."USD"') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 258e3f4caf08..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,34 +0,0 @@ - - create view _airbyte_test_normalization.`exchange_rate_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(currency as char) as currency, - case when `date` = '' then NULL - else cast(`date` as date) - end as `date` - , - cast(nullif(timestamp_col, '') as char) as timestamp_col, - cast(`HKD@spéçiäl & characters` as - float -) as `HKD@spéçiäl & characters`, - cast(hkd_special___characters as char) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`exchange_rate_ab1` --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index 6cc3a26b097f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view _airbyte_test_normalization.`exchange_rate_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_exchange_rate_hashid, - tmp.* -from _airbyte_test_normalization.`exchange_rate_ab2` tmp --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 7fd41343ac43..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,26 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - json_value(_airbyte_data, - '$."name"') as `name`, - json_value(_airbyte_data, - '$."_ab_cdc_lsn"') as _ab_cdc_lsn, - json_value(_airbyte_data, - '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, - json_value(_airbyte_data, - '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, - json_value(_airbyte_data, - '$."_ab_cdc_log_pos"') as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index 9f8c55f05dcf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - cast(`name` as char) as `name`, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`pos_dedup_cdcx_ab1` --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 446b1c167a26..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1__dbt_tmp` as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value(_airbyte_data, - '$."id"') as id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 01fe01259f91..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2__dbt_tmp` as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - signed -) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at -from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 4dd4da0f3431..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,74 +0,0 @@ - - - create table - test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` - as ( - -with - -input_data as ( - select * - from _airbyte_test_normalization.`dedup_cdc_excluded_ab3` - -- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char) - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), '')) as char)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 03eb062ccd38..6015fa2a1960 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -32,13 +32,13 @@ scd_data as ( `date` desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(`date`) over ( + case when row_number() over ( partition by id, currency, cast(nzd as char) order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 13bbe0a227a9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,66 +0,0 @@ - - - create table - test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` - as ( - -with - -input_data as ( - select * - from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` - -- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_renamed_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 37503d5bb636..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create table - test_normalization.`dedup_cdc_excluded__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from test_normalization.`dedup_cdc_excluded_scd` --- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded -where 1 = 1 -and _airbyte_active_row = 1 - - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 6b8a3751184f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - create table - test_normalization.`exchange_rate__dbt_tmp` - as ( - --- Final base SQL model -select - id, - currency, - `date`, - timestamp_col, - `HKD@spéçiäl & characters`, - hkd_special___characters, - nzd, - usd, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from _airbyte_test_normalization.`exchange_rate_ab3` --- exchange_rate from test_normalization._airbyte_raw_exchange_rate -where 1 = 1 - - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 10a547a0cccb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,22 +0,0 @@ - - - create table - test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from test_normalization.`renamed_dedup_cdc_excluded_scd` --- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded -where 1 = 1 -and _airbyte_active_row = 1 - - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index be6a9a952a26..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,76 +0,0 @@ - - - create table - test_normalization.`pos_dedup_cdcx_scd__dbt_tmp` - as ( - -with - -input_data as ( - select * - from _airbyte_test_normalization.`pos_dedup_cdcx_ab3` - -- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char), cast(_ab_cdc_log_pos as char) - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..f915bc6aceb6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,98 @@ + + + create table + test_normalization.`exchange_rate__dbt_tmp` + as ( + +with __dbt__CTE__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."currency"') as currency, + json_value(_airbyte_data, + '$."date"') as `date`, + json_value(_airbyte_data, + '$."timestamp_col"') as timestamp_col, + json_value(_airbyte_data, + '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + json_value(_airbyte_data, + '$."HKD_special___characters"') as hkd_special___characters, + json_value(_airbyte_data, + '$."NZD"') as nzd, + json_value(_airbyte_data, + '$."USD"') as usd, + json_value(_airbyte_data, + '$."column___with__quotes"') as `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__CTE__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(currency as char) as currency, + case when `date` = '' then NULL + else cast(`date` as date) + end as `date` + , + cast(nullif(timestamp_col, '') as char) as timestamp_col, + cast(`HKD@spéçiäl & characters` as + float +) as `HKD@spéçiäl & characters`, + cast(hkd_special___characters as char) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast(`column__'with"_quotes` as char) as `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__CTE__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), '')) as char)) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__CTE__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__CTE__exchange_rate_ab3 +-- exchange_rate from test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 736d35c45a9a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create table - test_normalization.`pos_dedup_cdcx__dbt_tmp` - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - - CURRENT_TIMESTAMP - as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from test_normalization.`pos_dedup_cdcx_scd` --- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and _airbyte_active_row = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 3caccb347235..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), '')) as char)) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from _airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 98129db90185..12e72758140b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,11 +1,72 @@ create view _airbyte_test_normalization.`dedup_exchange_rate_ab3__dbt_tmp` as ( --- SQL model to build a hash column based on the values of this record +with __dbt__CTE__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."currency"') as currency, + json_value(_airbyte_data, + '$."date"') as `date`, + json_value(_airbyte_data, + '$."timestamp_col"') as timestamp_col, + json_value(_airbyte_data, + '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + json_value(_airbyte_data, + '$."HKD_special___characters"') as hkd_special___characters, + json_value(_airbyte_data, + '$."NZD"') as nzd, + json_value(_airbyte_data, + '$."USD"') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__CTE__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(currency as char) as currency, + case when `date` = '' then NULL + else cast(`date` as date) + end as `date` + , + cast(nullif(timestamp_col, '') as char) as timestamp_col, + cast(`HKD@spéçiäl & characters` as + float +) as `HKD@spéçiäl & characters`, + cast(hkd_special___characters as char) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from __dbt__CTE__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_dedup_exchange_rate_hashid, tmp.* -from _airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +from __dbt__CTE__dedup_exchange_rate_ab2 tmp -- dedup_exchange_rate where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 857c87e6fc1d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,11 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from _airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 2df6fcdc3715..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,12 +0,0 @@ - - create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3__dbt_tmp` as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml new file mode 100755 index 000000000000..0ded2a42d60e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml @@ -0,0 +1,61 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: false + schema: false + identifier: false + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + # incremental is not enabled for Oracle yet + #+materialized: incremental + +materialized: table + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +vars: + dbt_utils_dispatch_list: ['airbyte_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index bd51b214ebc6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,93 +0,0 @@ - - - create table test_normalization.dedup_cdc_excluded_scd__dbt_tmp - - as - -with - -input_data as ( - select * - from test_normalization.dedup_cdc_excluded_ab3 - -- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - ora_hash( - - id - - ) as "_AIRBYTE_UNIQUE_KEY", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", - lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc - ) as "_AIRBYTE_END_AT", - case when lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc - ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)) - order by "_AIRBYTE_AB_ID" - ) as "_AIRBYTE_ROW_NUM", - ora_hash( - - "_AIRBYTE_UNIQUE_KEY" || '~' || - - - "_AIRBYTE_START_AT" || '~' || - - - "_AIRBYTE_EMITTED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" || '~' || - - - "_AB_CDC_UPDATED_AT" - - ) as "_AIRBYTE_UNIQUE_KEY_SCD", - scd_data.* - from scd_data -) -select - "_AIRBYTE_UNIQUE_KEY", - "_AIRBYTE_UNIQUE_KEY_SCD", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AIRBYTE_START_AT", - "_AIRBYTE_END_AT", - "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" -from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index e6c91980fea5..baf877bb0364 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -42,13 +42,13 @@ scd_data as ( "DATE" desc, "_AIRBYTE_EMITTED_AT" desc ) as "_AIRBYTE_END_AT", - case when lag("DATE") over ( + case when row_number() over ( partition by id, currency, cast(nzd as varchar2(4000)) order by "DATE" asc nulls last, "DATE" desc, "_AIRBYTE_EMITTED_AT" desc - ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + ) = 1 then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index c8d9914d17d2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,79 +0,0 @@ - - - create table test_normalization.renamed_dedup_cdc_excluded_scd__dbt_tmp - - as - -with - -input_data as ( - select * - from test_normalization.renamed_dedup_cdc_excluded_ab3 - -- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - ora_hash( - - id - - ) as "_AIRBYTE_UNIQUE_KEY", - id, - "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", - lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc - ) as "_AIRBYTE_END_AT", - case when lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc - ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" - order by "_AIRBYTE_AB_ID" - ) as "_AIRBYTE_ROW_NUM", - ora_hash( - - "_AIRBYTE_UNIQUE_KEY" || '~' || - - - "_AIRBYTE_START_AT" || '~' || - - - "_AIRBYTE_EMITTED_AT" - - ) as "_AIRBYTE_UNIQUE_KEY_SCD", - scd_data.* - from scd_data -) -select - "_AIRBYTE_UNIQUE_KEY", - "_AIRBYTE_UNIQUE_KEY_SCD", - id, - "_AIRBYTE_START_AT", - "_AIRBYTE_END_AT", - "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" -from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index bf4c8a11c07b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,24 +0,0 @@ - - - create table test_normalization.dedup_cdc_excluded__dbt_tmp - - as - --- Final base SQL model -select - "_AIRBYTE_UNIQUE_KEY", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" -from test_normalization.dedup_cdc_excluded_scd --- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded -where 1 = 1 -and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 9d77656b65a9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,114 +0,0 @@ - - - create table test_normalization.exchange_rate__dbt_tmp - - as - -with dbt__cte__exchange_rate_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."currency"') as currency, - json_value("_AIRBYTE_DATA", '$."date"') as "DATE", - json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, - json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, - json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, - json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, - json_value("_AIRBYTE_DATA", '$."USD"') as usd, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from test_normalization.airbyte_raw_exchange_rate --- exchange_rate -where 1 = 1 - -), dbt__cte__exchange_rate_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(currency as varchar2(4000)) as currency, - cast(nullif("DATE", '') as - varchar2(4000) -) as "DATE", - cast(nullif(timestamp_col, '') as - varchar2(4000) -) as timestamp_col, - cast(hkd_special___characters as - float -) as hkd_special___characters, - cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from dbt__cte__exchange_rate_ab1__ --- exchange_rate -where 1 = 1 - -), dbt__cte__exchange_rate_ab3__ as ( - --- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - currency || '~' || - - - "DATE" || '~' || - - - timestamp_col || '~' || - - - hkd_special___characters || '~' || - - - hkd_special___characters_1 || '~' || - - - nzd || '~' || - - - usd - - ) as "_AIRBYTE_EXCHANGE_RATE_HASHID", - tmp.* -from dbt__cte__exchange_rate_ab2__ tmp --- exchange_rate -where 1 = 1 - -)-- Final base SQL model -select - id, - currency, - "DATE", - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_EXCHANGE_RATE_HASHID" -from dbt__cte__exchange_rate_ab3__ --- exchange_rate from test_normalization.airbyte_raw_exchange_rate -where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index feac3368a6ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table test_normalization.renamed_dedup_cdc_excluded__dbt_tmp - - as - --- Final base SQL model -select - "_AIRBYTE_UNIQUE_KEY", - id, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" -from test_normalization.renamed_dedup_cdc_excluded_scd --- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded -where 1 = 1 -and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index c184fd31e2d3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,98 +0,0 @@ - - - create table test_normalization.pos_dedup_cdcx_scd__dbt_tmp - - as - -with - -input_data as ( - select * - from test_normalization.pos_dedup_cdcx_ab3 - -- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - ora_hash( - - id - - ) as "_AIRBYTE_UNIQUE_KEY", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AB_CDC_LOG_POS", - "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", - lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc - ) as "_AIRBYTE_END_AT", - case when lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc - ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_POS_DEDUP_CDCX_HASHID" - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)), cast("_AB_CDC_LOG_POS" as varchar2(4000)) - order by "_AIRBYTE_AB_ID" - ) as "_AIRBYTE_ROW_NUM", - ora_hash( - - "_AIRBYTE_UNIQUE_KEY" || '~' || - - - "_AIRBYTE_START_AT" || '~' || - - - "_AIRBYTE_EMITTED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" || '~' || - - - "_AB_CDC_UPDATED_AT" || '~' || - - - "_AB_CDC_LOG_POS" - - ) as "_AIRBYTE_UNIQUE_KEY_SCD", - scd_data.* - from scd_data -) -select - "_AIRBYTE_UNIQUE_KEY", - "_AIRBYTE_UNIQUE_KEY_SCD", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AB_CDC_LOG_POS", - "_AIRBYTE_START_AT", - "_AIRBYTE_END_AT", - "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_POS_DEDUP_CDCX_HASHID" -from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..37e9efe0d672 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,117 @@ + + + create table test_normalization.exchange_rate__dbt_tmp + + as + +with dbt__cte__exchange_rate_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."currency"') as currency, + json_value("_AIRBYTE_DATA", '$."date"') as "DATE", + json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, + json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, + json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, + json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, + json_value("_AIRBYTE_DATA", '$."USD"') as usd, + json_value("_AIRBYTE_DATA", '$."column___with__quotes"') as column___with__quotes, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_exchange_rate +-- exchange_rate +where 1 = 1 +), dbt__cte__exchange_rate_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(currency as varchar2(4000)) as currency, + cast(nullif("DATE", '') as + varchar2(4000) +) as "DATE", + cast(nullif(timestamp_col, '') as + varchar2(4000) +) as timestamp_col, + cast(hkd_special___characters as + float +) as hkd_special___characters, + cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast(column___with__quotes as varchar2(4000)) as column___with__quotes, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__exchange_rate_ab1__ +-- exchange_rate +where 1 = 1 +), dbt__cte__exchange_rate_ab3__ as ( + +-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + currency || '~' || + + + "DATE" || '~' || + + + timestamp_col || '~' || + + + hkd_special___characters || '~' || + + + hkd_special___characters_1 || '~' || + + + nzd || '~' || + + + usd || '~' || + + + column___with__quotes + + ) as "_AIRBYTE_EXCHANGE_RATE_HASHID", + tmp.* +from dbt__cte__exchange_rate_ab2__ tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + column___with__quotes, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_EXCHANGE_RATE_HASHID" +from dbt__cte__exchange_rate_ab3__ +-- exchange_rate from test_normalization.airbyte_raw_exchange_rate +where 1 = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index d3fd87ee5576..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - create table test_normalization.pos_dedup_cdcx__dbt_tmp - - as - --- Final base SQL model -select - "_AIRBYTE_UNIQUE_KEY", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AB_CDC_LOG_POS", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_POS_DEDUP_CDCX_HASHID" -from test_normalization.pos_dedup_cdcx_scd --- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 30135e3c86ec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,72 +0,0 @@ - - create view test_normalization.dedup_cdc_excluded_ab3__dbt_tmp as - -with dbt__cte__dedup_cdc_excluded_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."name"') as name, - json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from test_normalization.airbyte_raw_dedup_cdc_excluded --- dedup_cdc_excluded -where 1 = 1 - -), dbt__cte__dedup_cdc_excluded_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(name as varchar2(4000)) as name, - cast("_AB_CDC_LSN" as - float -) as "_AB_CDC_LSN", - cast("_AB_CDC_UPDATED_AT" as - float -) as "_AB_CDC_UPDATED_AT", - cast("_AB_CDC_DELETED_AT" as - float -) as "_AB_CDC_DELETED_AT", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from dbt__cte__dedup_cdc_excluded_ab1__ --- dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - name || '~' || - - - "_AB_CDC_LSN" || '~' || - - - "_AB_CDC_UPDATED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" - - ) as "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID", - tmp.* -from dbt__cte__dedup_cdc_excluded_ab2__ tmp --- dedup_cdc_excluded -where 1 = 1 - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index e1a0af83a632..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,76 +0,0 @@ - - create view test_normalization.pos_dedup_cdcx_ab3__dbt_tmp as - -with dbt__cte__pos_dedup_cdcx_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."name"') as name, - json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_log_pos"') as "_AB_CDC_LOG_POS", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from test_normalization.airbyte_raw_pos_dedup_cdcx --- pos_dedup_cdcx -where 1 = 1 -), dbt__cte__pos_dedup_cdcx_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(name as varchar2(4000)) as name, - cast("_AB_CDC_LSN" as - float -) as "_AB_CDC_LSN", - cast("_AB_CDC_UPDATED_AT" as - float -) as "_AB_CDC_UPDATED_AT", - cast("_AB_CDC_DELETED_AT" as - float -) as "_AB_CDC_DELETED_AT", - cast("_AB_CDC_LOG_POS" as - float -) as "_AB_CDC_LOG_POS", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from dbt__cte__pos_dedup_cdcx_ab1__ --- pos_dedup_cdcx -where 1 = 1 -)-- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - name || '~' || - - - "_AB_CDC_LSN" || '~' || - - - "_AB_CDC_UPDATED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" || '~' || - - - "_AB_CDC_LOG_POS" - - ) as "_AIRBYTE_POS_DEDUP_CDCX_HASHID", - tmp.* -from dbt__cte__pos_dedup_cdcx_ab2__ tmp --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 6ad67cd02a90..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,46 +0,0 @@ - - create view test_normalization.renamed_dedup_cdc_excluded_ab3__dbt_tmp as - -with dbt__cte__renamed_dedup_cdc_excluded_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded --- renamed_dedup_cdc_excluded -where 1 = 1 - -), dbt__cte__renamed_dedup_cdc_excluded_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from dbt__cte__renamed_dedup_cdc_excluded_ab1__ --- renamed_dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id - - ) as "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID", - tmp.* -from dbt__cte__renamed_dedup_cdc_excluded_ab2__ tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 2f3b6d1b7104..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['name'], ['name']) }} as name, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as {{ quote('_AB_CDC_LSN') }}, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} -from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 9f7cb197462d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(name as {{ dbt_utils.type_string() }}) as name, - cast({{ quote('_AB_CDC_LSN') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_LSN') }}, - cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_UPDATED_AT') }}, - cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} -from {{ ref('dedup_cdc_excluded_ab1') }} --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index a0dda141cd02..83775cff3a21 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + unique_key = quote('_AIRBYTE_AB_ID'), schema = "test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index fadc49c40be3..485d3e886480 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + unique_key = quote('_AIRBYTE_AB_ID'), schema = "test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index bcc1db00d8e3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['currency'], ['currency']) }} as currency, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['date'], ['date']) }} as {{ quote('DATE') }}, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as hkd_special___characters, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters_1, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['NZD'], ['NZD']) }} as nzd, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['USD'], ['USD']) }} as usd, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} -from {{ source('test_normalization', 'airbyte_raw_exchange_rate') }} --- exchange_rate -where 1 = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index e7f5e3353cfd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - cast({{ empty_string_to_null(quote('DATE')) }} as {{ type_date() }}) as {{ quote('DATE') }}, - cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col, - cast(hkd_special___characters as {{ dbt_utils.type_float() }}) as hkd_special___characters, - cast(hkd_special___characters_1 as {{ dbt_utils.type_string() }}) as hkd_special___characters_1, - cast(nzd as {{ dbt_utils.type_float() }}) as nzd, - cast(usd as {{ dbt_utils.type_float() }}) as usd, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} -from {{ ref('exchange_rate_ab1') }} --- exchange_rate -where 1 = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index 21fb107c1c44..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - quote('DATE'), - 'timestamp_col', - 'hkd_special___characters', - 'hkd_special___characters_1', - 'nzd', - 'usd', - ]) }} as {{ quote('_AIRBYTE_EXCHANGE_RATE_HASHID') }}, - tmp.* -from {{ ref('exchange_rate_ab2') }} tmp --- exchange_rate -where 1 = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 9cb45ceefec4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['name'], ['name']) }} as name, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as {{ quote('_AB_CDC_LSN') }}, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as {{ quote('_AB_CDC_DELETED_AT') }}, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} -from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index de537cd43138..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(name as {{ dbt_utils.type_string() }}) as name, - cast({{ quote('_AB_CDC_LSN') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_LSN') }}, - cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_UPDATED_AT') }}, - cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_DELETED_AT') }}, - cast({{ quote('_AB_CDC_LOG_POS') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} -from {{ ref('pos_dedup_cdcx_ab1') }} --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index e7fe7d6f462c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} -from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index fc3191e47f30..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} -from {{ ref('renamed_dedup_cdc_excluded_ab1') }} --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 3f81f2997f7c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,111 +0,0 @@ -{{ config( - unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('dedup_cdc_excluded_ab3') }} - -- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} - where 1 = 1 - {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} -), -new_data_ids as ( - -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 - where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('dedup_cdc_excluded_ab3') }} - -- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, - id, - name, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, - lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by - {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc - ) as {{ quote('_AIRBYTE_END_AT') }}, - case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by - {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc - ) is null and {{ quote('_AB_CDC_DELETED_AT') }} is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_string() }}), cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_string() }}) - order by {{ quote('_AIRBYTE_AB_ID') }} - ) as {{ quote('_AIRBYTE_ROW_NUM') }}, - {{ dbt_utils.surrogate_key([ - quote('_AIRBYTE_UNIQUE_KEY'), - quote('_AIRBYTE_START_AT'), - quote('_AIRBYTE_EMITTED_AT'), quote('_AB_CDC_DELETED_AT'), quote('_AB_CDC_UPDATED_AT') - ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, - scd_data.* - from scd_data -) -select - {{ quote('_AIRBYTE_UNIQUE_KEY') }}, - {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, - id, - name, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_START_AT') }}, - {{ quote('_AIRBYTE_END_AT') }}, - {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, - {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} -from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 6ad3983885b4..743bf823d1ac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -71,13 +71,13 @@ scd_data as ( {{ quote('DATE') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc ) as {{ quote('_AIRBYTE_END_AT') }}, - case when lag({{ quote('DATE') }}) over ( + case when row_number() over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by {{ quote('DATE') }} asc nulls last, {{ quote('DATE') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + ) = 1 then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, {{ quote('_AIRBYTE_AB_ID') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 9d79d2cbac72..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,103 +0,0 @@ -{{ config( - unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} - -- renamed_dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} - where 1 = 1 - {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} -), -new_data_ids as ( - -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 - where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} - -- renamed_dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, - id, - {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, - lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by - {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) as {{ quote('_AIRBYTE_END_AT') }}, - case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by - {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }} - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }} - order by {{ quote('_AIRBYTE_AB_ID') }} - ) as {{ quote('_AIRBYTE_ROW_NUM') }}, - {{ dbt_utils.surrogate_key([ - quote('_AIRBYTE_UNIQUE_KEY'), - quote('_AIRBYTE_START_AT'), - quote('_AIRBYTE_EMITTED_AT') - ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, - scd_data.* - from scd_data -) -select - {{ quote('_AIRBYTE_UNIQUE_KEY') }}, - {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, - id, - {{ quote('_AIRBYTE_START_AT') }}, - {{ quote('_AIRBYTE_END_AT') }}, - {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, - {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }} -from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 20be5269fbc9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - {{ quote('_AIRBYTE_UNIQUE_KEY') }}, - id, - name, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, - {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} -where 1 = 1 -and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index c87e205f2a32..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - currency, - {{ quote('DATE') }}, - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, - {{ quote('_AIRBYTE_EXCHANGE_RATE_HASHID') }} -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', 'airbyte_raw_exchange_rate') }} -where 1 = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 40bf1d0d0013..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - {{ quote('_AIRBYTE_UNIQUE_KEY') }}, - id, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, - {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }} -from {{ ref('renamed_dedup_cdc_excluded_scd') }} --- renamed_dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} -where 1 = 1 -and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 506d15538add..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,113 +0,0 @@ -{{ config( - unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} - where 1 = 1 - {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} -), -new_data_ids as ( - -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 - where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, - id, - name, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, - lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by - {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc, {{ quote('_AB_CDC_LOG_POS') }} desc - ) as {{ quote('_AIRBYTE_END_AT') }}, - case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by - {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc, - {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc, {{ quote('_AB_CDC_LOG_POS') }} desc - ) is null and {{ quote('_AB_CDC_DELETED_AT') }} is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_string() }}), cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_string() }}), cast({{ quote('_AB_CDC_LOG_POS') }} as {{ dbt_utils.type_string() }}) - order by {{ quote('_AIRBYTE_AB_ID') }} - ) as {{ quote('_AIRBYTE_ROW_NUM') }}, - {{ dbt_utils.surrogate_key([ - quote('_AIRBYTE_UNIQUE_KEY'), - quote('_AIRBYTE_START_AT'), - quote('_AIRBYTE_EMITTED_AT'), quote('_AB_CDC_DELETED_AT'), quote('_AB_CDC_UPDATED_AT'), quote('_AB_CDC_LOG_POS') - ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, - scd_data.* - from scd_data -) -select - {{ quote('_AIRBYTE_UNIQUE_KEY') }}, - {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, - id, - name, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_START_AT') }}, - {{ quote('_AIRBYTE_END_AT') }}, - {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, - {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} -from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..83ccd89c14c2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,24 @@ +{{ config( + unique_key = quote('_AIRBYTE_AB_ID'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + {{ quote('DATE') }}, + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + column___with__quotes, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_EXCHANGE_RATE_HASHID') }} +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', 'airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 26ed51d88c7b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,23 +0,0 @@ -{{ config( - unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - {{ quote('_AIRBYTE_UNIQUE_KEY') }}, - id, - name, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_AB_ID') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, - {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} -from {{ ref('pos_dedup_cdcx_scd') }} --- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} -where 1 = 1 -and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index f9575722cc41..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - quote('_AB_CDC_LSN'), - quote('_AB_CDC_UPDATED_AT'), - quote('_AB_CDC_DELETED_AT'), - ]) }} as {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }}, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 48905c777a2d..b648997f9a27 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,5 +1,5 @@ {{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + unique_key = quote('_AIRBYTE_AB_ID'), schema = "test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 52cf027c3895..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - quote('_AB_CDC_LSN'), - quote('_AB_CDC_UPDATED_AT'), - quote('_AB_CDC_DELETED_AT'), - quote('_AB_CDC_LOG_POS'), - ]) }} as {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }}, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 7555ff5d1efd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config( - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), - schema = "test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }}, - tmp.* -from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index bd51b214ebc6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,93 +0,0 @@ - - - create table test_normalization.dedup_cdc_excluded_scd__dbt_tmp - - as - -with - -input_data as ( - select * - from test_normalization.dedup_cdc_excluded_ab3 - -- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - ora_hash( - - id - - ) as "_AIRBYTE_UNIQUE_KEY", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", - lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc - ) as "_AIRBYTE_END_AT", - case when lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc - ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)) - order by "_AIRBYTE_AB_ID" - ) as "_AIRBYTE_ROW_NUM", - ora_hash( - - "_AIRBYTE_UNIQUE_KEY" || '~' || - - - "_AIRBYTE_START_AT" || '~' || - - - "_AIRBYTE_EMITTED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" || '~' || - - - "_AB_CDC_UPDATED_AT" - - ) as "_AIRBYTE_UNIQUE_KEY_SCD", - scd_data.* - from scd_data -) -select - "_AIRBYTE_UNIQUE_KEY", - "_AIRBYTE_UNIQUE_KEY_SCD", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AIRBYTE_START_AT", - "_AIRBYTE_END_AT", - "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" -from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index e6c91980fea5..baf877bb0364 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -42,13 +42,13 @@ scd_data as ( "DATE" desc, "_AIRBYTE_EMITTED_AT" desc ) as "_AIRBYTE_END_AT", - case when lag("DATE") over ( + case when row_number() over ( partition by id, currency, cast(nzd as varchar2(4000)) order by "DATE" asc nulls last, "DATE" desc, "_AIRBYTE_EMITTED_AT" desc - ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + ) = 1 then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index c8d9914d17d2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,79 +0,0 @@ - - - create table test_normalization.renamed_dedup_cdc_excluded_scd__dbt_tmp - - as - -with - -input_data as ( - select * - from test_normalization.renamed_dedup_cdc_excluded_ab3 - -- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - ora_hash( - - id - - ) as "_AIRBYTE_UNIQUE_KEY", - id, - "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", - lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc - ) as "_AIRBYTE_END_AT", - case when lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc - ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" - order by "_AIRBYTE_AB_ID" - ) as "_AIRBYTE_ROW_NUM", - ora_hash( - - "_AIRBYTE_UNIQUE_KEY" || '~' || - - - "_AIRBYTE_START_AT" || '~' || - - - "_AIRBYTE_EMITTED_AT" - - ) as "_AIRBYTE_UNIQUE_KEY_SCD", - scd_data.* - from scd_data -) -select - "_AIRBYTE_UNIQUE_KEY", - "_AIRBYTE_UNIQUE_KEY_SCD", - id, - "_AIRBYTE_START_AT", - "_AIRBYTE_END_AT", - "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" -from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index bf4c8a11c07b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,24 +0,0 @@ - - - create table test_normalization.dedup_cdc_excluded__dbt_tmp - - as - --- Final base SQL model -select - "_AIRBYTE_UNIQUE_KEY", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" -from test_normalization.dedup_cdc_excluded_scd --- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded -where 1 = 1 -and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 9d77656b65a9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,114 +0,0 @@ - - - create table test_normalization.exchange_rate__dbt_tmp - - as - -with dbt__cte__exchange_rate_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."currency"') as currency, - json_value("_AIRBYTE_DATA", '$."date"') as "DATE", - json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, - json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, - json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, - json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, - json_value("_AIRBYTE_DATA", '$."USD"') as usd, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from test_normalization.airbyte_raw_exchange_rate --- exchange_rate -where 1 = 1 - -), dbt__cte__exchange_rate_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(currency as varchar2(4000)) as currency, - cast(nullif("DATE", '') as - varchar2(4000) -) as "DATE", - cast(nullif(timestamp_col, '') as - varchar2(4000) -) as timestamp_col, - cast(hkd_special___characters as - float -) as hkd_special___characters, - cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from dbt__cte__exchange_rate_ab1__ --- exchange_rate -where 1 = 1 - -), dbt__cte__exchange_rate_ab3__ as ( - --- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - currency || '~' || - - - "DATE" || '~' || - - - timestamp_col || '~' || - - - hkd_special___characters || '~' || - - - hkd_special___characters_1 || '~' || - - - nzd || '~' || - - - usd - - ) as "_AIRBYTE_EXCHANGE_RATE_HASHID", - tmp.* -from dbt__cte__exchange_rate_ab2__ tmp --- exchange_rate -where 1 = 1 - -)-- Final base SQL model -select - id, - currency, - "DATE", - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_EXCHANGE_RATE_HASHID" -from dbt__cte__exchange_rate_ab3__ --- exchange_rate from test_normalization.airbyte_raw_exchange_rate -where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index feac3368a6ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create table test_normalization.renamed_dedup_cdc_excluded__dbt_tmp - - as - --- Final base SQL model -select - "_AIRBYTE_UNIQUE_KEY", - id, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" -from test_normalization.renamed_dedup_cdc_excluded_scd --- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded -where 1 = 1 -and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index c184fd31e2d3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,98 +0,0 @@ - - - create table test_normalization.pos_dedup_cdcx_scd__dbt_tmp - - as - -with - -input_data as ( - select * - from test_normalization.pos_dedup_cdcx_ab3 - -- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - ora_hash( - - id - - ) as "_AIRBYTE_UNIQUE_KEY", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AB_CDC_LOG_POS", - "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", - lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc - ) as "_AIRBYTE_END_AT", - case when lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by - "_AIRBYTE_EMITTED_AT" asc nulls last, - "_AIRBYTE_EMITTED_AT" desc, - "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc - ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_POS_DEDUP_CDCX_HASHID" - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)), cast("_AB_CDC_LOG_POS" as varchar2(4000)) - order by "_AIRBYTE_AB_ID" - ) as "_AIRBYTE_ROW_NUM", - ora_hash( - - "_AIRBYTE_UNIQUE_KEY" || '~' || - - - "_AIRBYTE_START_AT" || '~' || - - - "_AIRBYTE_EMITTED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" || '~' || - - - "_AB_CDC_UPDATED_AT" || '~' || - - - "_AB_CDC_LOG_POS" - - ) as "_AIRBYTE_UNIQUE_KEY_SCD", - scd_data.* - from scd_data -) -select - "_AIRBYTE_UNIQUE_KEY", - "_AIRBYTE_UNIQUE_KEY_SCD", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AB_CDC_LOG_POS", - "_AIRBYTE_START_AT", - "_AIRBYTE_END_AT", - "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_POS_DEDUP_CDCX_HASHID" -from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..37e9efe0d672 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,117 @@ + + + create table test_normalization.exchange_rate__dbt_tmp + + as + +with dbt__cte__exchange_rate_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."currency"') as currency, + json_value("_AIRBYTE_DATA", '$."date"') as "DATE", + json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, + json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, + json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, + json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, + json_value("_AIRBYTE_DATA", '$."USD"') as usd, + json_value("_AIRBYTE_DATA", '$."column___with__quotes"') as column___with__quotes, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_exchange_rate +-- exchange_rate +where 1 = 1 +), dbt__cte__exchange_rate_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(currency as varchar2(4000)) as currency, + cast(nullif("DATE", '') as + varchar2(4000) +) as "DATE", + cast(nullif(timestamp_col, '') as + varchar2(4000) +) as timestamp_col, + cast(hkd_special___characters as + float +) as hkd_special___characters, + cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast(column___with__quotes as varchar2(4000)) as column___with__quotes, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__exchange_rate_ab1__ +-- exchange_rate +where 1 = 1 +), dbt__cte__exchange_rate_ab3__ as ( + +-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + currency || '~' || + + + "DATE" || '~' || + + + timestamp_col || '~' || + + + hkd_special___characters || '~' || + + + hkd_special___characters_1 || '~' || + + + nzd || '~' || + + + usd || '~' || + + + column___with__quotes + + ) as "_AIRBYTE_EXCHANGE_RATE_HASHID", + tmp.* +from dbt__cte__exchange_rate_ab2__ tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + column___with__quotes, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_EXCHANGE_RATE_HASHID" +from dbt__cte__exchange_rate_ab3__ +-- exchange_rate from test_normalization.airbyte_raw_exchange_rate +where 1 = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index d3fd87ee5576..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - create table test_normalization.pos_dedup_cdcx__dbt_tmp - - as - --- Final base SQL model -select - "_AIRBYTE_UNIQUE_KEY", - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AB_CDC_LOG_POS", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT", - "_AIRBYTE_POS_DEDUP_CDCX_HASHID" -from test_normalization.pos_dedup_cdcx_scd --- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 30135e3c86ec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,72 +0,0 @@ - - create view test_normalization.dedup_cdc_excluded_ab3__dbt_tmp as - -with dbt__cte__dedup_cdc_excluded_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."name"') as name, - json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from test_normalization.airbyte_raw_dedup_cdc_excluded --- dedup_cdc_excluded -where 1 = 1 - -), dbt__cte__dedup_cdc_excluded_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(name as varchar2(4000)) as name, - cast("_AB_CDC_LSN" as - float -) as "_AB_CDC_LSN", - cast("_AB_CDC_UPDATED_AT" as - float -) as "_AB_CDC_UPDATED_AT", - cast("_AB_CDC_DELETED_AT" as - float -) as "_AB_CDC_DELETED_AT", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from dbt__cte__dedup_cdc_excluded_ab1__ --- dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - name || '~' || - - - "_AB_CDC_LSN" || '~' || - - - "_AB_CDC_UPDATED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" - - ) as "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID", - tmp.* -from dbt__cte__dedup_cdc_excluded_ab2__ tmp --- dedup_cdc_excluded -where 1 = 1 - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index e1a0af83a632..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,76 +0,0 @@ - - create view test_normalization.pos_dedup_cdcx_ab3__dbt_tmp as - -with dbt__cte__pos_dedup_cdcx_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."name"') as name, - json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_log_pos"') as "_AB_CDC_LOG_POS", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from test_normalization.airbyte_raw_pos_dedup_cdcx --- pos_dedup_cdcx -where 1 = 1 -), dbt__cte__pos_dedup_cdcx_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(name as varchar2(4000)) as name, - cast("_AB_CDC_LSN" as - float -) as "_AB_CDC_LSN", - cast("_AB_CDC_UPDATED_AT" as - float -) as "_AB_CDC_UPDATED_AT", - cast("_AB_CDC_DELETED_AT" as - float -) as "_AB_CDC_DELETED_AT", - cast("_AB_CDC_LOG_POS" as - float -) as "_AB_CDC_LOG_POS", - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from dbt__cte__pos_dedup_cdcx_ab1__ --- pos_dedup_cdcx -where 1 = 1 -)-- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - name || '~' || - - - "_AB_CDC_LSN" || '~' || - - - "_AB_CDC_UPDATED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" || '~' || - - - "_AB_CDC_LOG_POS" - - ) as "_AIRBYTE_POS_DEDUP_CDCX_HASHID", - tmp.* -from dbt__cte__pos_dedup_cdcx_ab2__ tmp --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 6ad67cd02a90..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,46 +0,0 @@ - - create view test_normalization.renamed_dedup_cdc_excluded_ab3__dbt_tmp as - -with dbt__cte__renamed_dedup_cdc_excluded_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded --- renamed_dedup_cdc_excluded -where 1 = 1 - -), dbt__cte__renamed_dedup_cdc_excluded_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - "_AIRBYTE_AB_ID", - "_AIRBYTE_EMITTED_AT", - - CURRENT_TIMESTAMP - as "_AIRBYTE_NORMALIZED_AT" -from dbt__cte__renamed_dedup_cdc_excluded_ab1__ --- renamed_dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id - - ) as "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID", - tmp.* -from dbt__cte__renamed_dedup_cdc_excluded_ab2__ tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml new file mode 100755 index 000000000000..9ad815875900 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index ce2e7a3ec5b5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path(_airbyte_data, 'conflict_stream_array') as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index a4df9ebe22b0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."conflict_stream_array_ab1" --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index de392f8c307b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(conflict_stream_array as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_array_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_array_ab2" tmp --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql deleted file mode 100644 index da03bc0bb8db..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - jsonb_extract_path_text(conflict_stream_name, 'groups') as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql deleted file mode 100644 index f40efe7753f1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as - varchar -) as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql deleted file mode 100644 index 585ae9060bfe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as - varchar -), '') || '-' || coalesce(cast(groups as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index edec407e9552..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - - jsonb_extract_path(table_alias._airbyte_data, 'conflict_stream_name') - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index fed1290ad9a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast(conflict_stream_name as - jsonb -) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."conflict_stream_name_ab1" --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index d82b4294172d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(conflict_stream_name as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_name_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_name_ab2" tmp --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index f0ca905a6656..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - jsonb_extract_path(table_alias.conflict_stream_name, 'conflict_stream_name') - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index a3a5870bec5d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as - jsonb -) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index 8435a2464b7b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as - varchar -), '') || '-' || coalesce(cast(conflict_stream_name as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index abb9d5899986..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'conflict_stream_scalar') as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index a5a94a9cb6c1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast(conflict_stream_scalar as - bigint -) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1" --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 66ad40a66f43..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(conflict_stream_scalar as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql deleted file mode 100644 index f6e2700418b3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_stre__nto_long_names_hashid, - jsonb_extract_path("partition", 'double_array_data') as double_array_data, - jsonb_extract_path("partition", 'DATA') as "DATA", - jsonb_extract_path("partition", 'column`_''with"_quotes') as "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and "partition" is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql deleted file mode 100644 index 451bd82174e7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stre__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql deleted file mode 100644 index eb31fad896ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_nested_stre__nto_long_names_hashid as - varchar -), '') || '-' || coalesce(cast(double_array_data as - varchar -), '') || '-' || coalesce(cast("DATA" as - varchar -), '') || '-' || coalesce(cast("column`_'with""_quotes" as - varchar -), '') as - varchar -)) as _airbyte_partition_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2" tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql deleted file mode 100644 index 602dad2547f8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -cross join jsonb_array_elements( - case jsonb_typeof("DATA") - when 'array' then "DATA" - else '[]' end - ) as _airbyte_nested_data -where 1 = 1 -and "DATA" is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql deleted file mode 100644 index c51fbcb9e61f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - varchar -) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql deleted file mode 100644 index 94e1500a8681..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') as - varchar -)) as _airbyte_data_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2" tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql deleted file mode 100644 index d6e58ff5015a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -cross join jsonb_array_elements( - case jsonb_typeof("column`_'with""_quotes") - when 'array' then "column`_'with""_quotes" - else '[]' end - ) as _airbyte_nested_data -where 1 = 1 -and "column`_'with""_quotes" is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql deleted file mode 100644 index 3517ce920c9a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - varchar -) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql deleted file mode 100644 index 37d739b12c2c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') as - varchar -)) as _airbyte_column___with__quotes_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2" tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql deleted file mode 100644 index e18ebddd1a8c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'id') as "id", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -cross join jsonb_array_elements( - case jsonb_typeof(double_array_data) - when 'array' then double_array_data - else '[]' end - ) as _airbyte_nested_data -where 1 = 1 -and double_array_data is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql deleted file mode 100644 index d716b1eb2f5d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast("id" as - varchar -) as "id", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql deleted file mode 100644 index f4511754ecfc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as - varchar -), '') || '-' || coalesce(cast("id" as - varchar -), '') as - varchar -)) as _airbyte_double_array_data_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2" tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql deleted file mode 100644 index fd8bdeb88d0c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - - jsonb_extract_path(table_alias._airbyte_data, 'partition') - as "partition", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_c__lting_into_long_names -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql deleted file mode 100644 index 3eacd30f282f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,22 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast("date" as - varchar -) as "date", - cast("partition" as - jsonb -) as "partition", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1" --- nested_stream_with_c__lting_into_long_names -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql deleted file mode 100644 index 2a1e4ca28f20..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_wi__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql deleted file mode 100644 index 51620aee9290..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast("date" as - varchar -) as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1" --- non_nested_stream_wi__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql deleted file mode 100644 index 151cdb6e10a3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') as - varchar -)) as _airbyte_non_nested___nto_long_names_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2" tmp --- non_nested_stream_wi__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index b70f91add752..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path(_airbyte_data, 'children') as children, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index 86b9fb2c93f3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - children, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."unnest_alias_ab1" --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index f23e4c1aca53..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(children as - varchar -), '') as - varchar -)) as _airbyte_unnest_alias_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."unnest_alias_ab2" tmp --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 04ce3aa2eacc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_unnest_alias_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'ab_id') as ab_id, - - jsonb_extract_path(_airbyte_nested_data, 'owner') - as "owner", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."unnest_alias" as table_alias --- children at unnest_alias/children -cross join jsonb_array_elements( - case jsonb_typeof(children) - when 'array' then children - else '[]' end - ) as _airbyte_nested_data -where 1 = 1 -and children is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index 00521425f33e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - bigint -) as ab_id, - cast("owner" as - jsonb -) as "owner", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."unnest_alias_children_ab1" --- children at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index f11f03175e3b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as - varchar -), '') || '-' || coalesce(cast(ab_id as - varchar -), '') || '-' || coalesce(cast("owner" as - varchar -), '') as - varchar -)) as _airbyte_children_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."unnest_alias_children_ab2" tmp --- children at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 3aafcf647131..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - jsonb_extract_path_text("owner", 'owner_id') as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."unnest_alias_children" as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and "owner" is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index b2ca5615ca90..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - bigint -) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1" --- owner at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 8239aba10289..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_children_hashid as - varchar -), '') || '-' || coalesce(cast(owner_id as - varchar -), '') as - varchar -)) as _airbyte_owner_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp --- owner at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql deleted file mode 100644 index 5211a6eeff16..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_n__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql deleted file mode 100644 index 8f67edcf78c3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast("date" as - varchar -) as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1" --- simple_stream_with_n__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql deleted file mode 100644 index 1c204c3383bb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') as - varchar -)) as _airbyte_simple_stre__nto_long_names_hashid, - tmp.* -from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2" tmp --- simple_stream_with_n__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql index d844645b4961..b3c34afd1712 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -31,13 +31,13 @@ scd_data as ( "date" desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag("date") over ( + case when row_number() over ( partition by "id" order by "date" is null asc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_stre__nto_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c___long_names_partition.sql new file mode 100644 index 000000000000..d59d4d25ca54 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c___long_names_partition.sql @@ -0,0 +1,68 @@ + + + + create table "postgres".test_normalization."nested_stream_with_c___long_names_partition" + as ( + +with __dbt__cte__nested_stream_with_c___long_names_partition_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stre__nto_long_names_hashid, + jsonb_extract_path("partition", 'double_array_data') as double_array_data, + jsonb_extract_path("partition", 'DATA') as "DATA", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null + +), __dbt__cte__nested_stream_with_c___long_names_partition_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stre__nto_long_names_hashid, + double_array_data, + "DATA", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_c___long_names_partition_ab1 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +), __dbt__cte__nested_stream_with_c___long_names_partition_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_nested_stre__nto_long_names_hashid as + varchar +), '') || '-' || coalesce(cast(double_array_data as + varchar +), '') || '-' || coalesce(cast("DATA" as + varchar +), '') as + varchar +)) as _airbyte_partition_hashid, + tmp.* +from __dbt__cte__nested_stream_with_c___long_names_partition_ab2 tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_nested_stre__nto_long_names_hashid, + double_array_data, + "DATA", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_partition_hashid +from __dbt__cte__nested_stream_with_c___long_names_partition_ab3 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c___names_partition_data.sql new file mode 100644 index 000000000000..1ea5e32d4ad6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c___names_partition_data.sql @@ -0,0 +1,71 @@ + + + + create table "postgres".test_normalization."nested_stream_with_c___names_partition_data" + as ( + +with __dbt__cte__nested_stream_with_c___names_partition_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join jsonb_array_elements( + case jsonb_typeof("DATA") + when 'array' then "DATA" + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and "DATA" is not null + +), __dbt__cte__nested_stream_with_c___names_partition_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + varchar +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_c___names_partition_data_ab1 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +), __dbt__cte__nested_stream_with_c___names_partition_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') as + varchar +)) as _airbyte_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_c___names_partition_data_ab2 tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_data_hashid +from __dbt__cte__nested_stream_with_c___names_partition_data_ab3 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__ion_double_array_data.sql new file mode 100644 index 000000000000..ad4c416a65df --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__ion_double_array_data.sql @@ -0,0 +1,71 @@ + + + + create table "postgres".test_normalization."nested_stream_with_c__ion_double_array_data" + as ( + +with __dbt__cte__nested_stream_with_c__ion_double_array_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'id') as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join jsonb_array_elements( + case jsonb_typeof(double_array_data) + when 'array' then double_array_data + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and double_array_data is not null + +), __dbt__cte__nested_stream_with_c__ion_double_array_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast("id" as + varchar +) as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_c__ion_double_array_data_ab1 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +), __dbt__cte__nested_stream_with_c__ion_double_array_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_double_array_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_c__ion_double_array_data_ab2 tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from __dbt__cte__nested_stream_with_c__ion_double_array_data_ab3 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..a66bd29ad0a4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql @@ -0,0 +1,60 @@ + + + + create table "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3" + as ( + +with __dbt__cte__nested_stream_with_c__lting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + + jsonb_extract_path(table_alias._airbyte_data, 'partition') + as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + +), __dbt__cte__nested_stream_with_c__lting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + cast("partition" as + jsonb +) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_c__lting_into_long_names_ab1 +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast("partition" as + varchar +), '') as + varchar +)) as _airbyte_nested_stre__nto_long_names_hashid, + tmp.* +from __dbt__cte__nested_stream_with_c__lting_into_long_names_ab2 tmp +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index b1be69446486..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - - create table "postgres".test_normalization."unnest_alias" - as ( - --- Final base SQL model -select - "id", - children, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from "postgres"._airbyte_test_normalization."unnest_alias_ab3" --- unnest_alias from "postgres".test_normalization._airbyte_raw_unnest_alias -where 1 = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql new file mode 100644 index 000000000000..cd54a42ac9f3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql @@ -0,0 +1,66 @@ + + + + create table "postgres".test_normalization_namespace."simple_stream_with_n__lting_into_long_names" + as ( + +with __dbt__cte__simple_stream_with_n__lting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + +), __dbt__cte__simple_stream_with_n__lting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__simple_stream_with_n__lting_into_long_names_ab1 +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + +), __dbt__cte__simple_stream_with_n__lting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') as + varchar +)) as _airbyte_simple_stre__nto_long_names_hashid, + tmp.* +from __dbt__cte__simple_stream_with_n__lting_into_long_names_ab2 tmp +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + +)-- Final base SQL model +select + "id", + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_simple_stre__nto_long_names_hashid +from __dbt__cte__simple_stream_with_n__lting_into_long_names_ab3 +-- simple_stream_with_n__lting_into_long_names from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql index 24d04f946c89..ab5935b1d289 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -3,7 +3,48 @@ create table "postgres".test_normalization."conflict_stream_array__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__conflict_stream_array_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path(_airbyte_data, 'conflict_stream_array') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 +), __dbt__cte__conflict_stream_array_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_array_ab1 +-- conflict_stream_array +where 1 = 1 +), __dbt__cte__conflict_stream_array_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_array as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from __dbt__cte__conflict_stream_array_ab2 tmp +-- conflict_stream_array +where 1 = 1 +)-- Final base SQL model select "id", conflict_stream_array, @@ -11,7 +52,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_array_ab3" +from __dbt__cte__conflict_stream_array_ab3 -- conflict_stream_array from "postgres".test_normalization._airbyte_raw_conflict_stream_array where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql index a664bb56d781..8b379cafff07 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -3,7 +3,52 @@ create table "postgres".test_normalization."conflict_stream_name__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + + jsonb_extract_path(table_alias._airbyte_data, 'conflict_stream_name') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast(conflict_stream_name as + jsonb +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_ab1 +-- conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_name as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_ab2 tmp +-- conflict_stream_name +where 1 = 1 +)-- Final base SQL model select "id", conflict_stream_name, @@ -11,7 +56,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_name_ab3" +from __dbt__cte__conflict_stream_name_ab3 -- conflict_stream_name from "postgres".test_normalization._airbyte_raw_conflict_stream_name where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql index 0e06a3243fa2..1729627b03eb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql @@ -3,7 +3,49 @@ create table "postgres".test_normalization."conflict_stream_name___conflict_stream_name__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__conflict_stream_name___conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + jsonb_extract_path_text(conflict_stream_name, 'groups') as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null +), __dbt__cte__conflict_stream_name___conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as + varchar +) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name___conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name___conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + varchar +), '') || '-' || coalesce(cast(groups as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from __dbt__cte__conflict_stream_name___conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +)-- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, groups, @@ -11,7 +53,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3" +from __dbt__cte__conflict_stream_name___conflict_stream_name_ab3 -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index df6e327e1620..dc9657144317 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -3,7 +3,51 @@ create table "postgres".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + jsonb_extract_path(table_alias.conflict_stream_name, 'conflict_stream_name') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null +), __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as + jsonb +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as + varchar +), '') || '-' || coalesce(cast(conflict_stream_name as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +)-- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, @@ -11,7 +55,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 -- conflict_stream_name at conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name" where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql index bdcd7a2664c1..2e06b81d6a56 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -3,7 +3,50 @@ create table "postgres".test_normalization."conflict_stream_scalar__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__conflict_stream_scalar_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'conflict_stream_scalar') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 +), __dbt__cte__conflict_stream_scalar_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_scalar_ab1 +-- conflict_stream_scalar +where 1 = 1 +), __dbt__cte__conflict_stream_scalar_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_scalar as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from __dbt__cte__conflict_stream_scalar_ab2 tmp +-- conflict_stream_scalar +where 1 = 1 +)-- Final base SQL model select "id", conflict_stream_scalar, @@ -11,7 +54,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3" +from __dbt__cte__conflict_stream_scalar_ab3 -- conflict_stream_scalar from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql deleted file mode 100644 index a9781fcf92d2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c___long_names_partition__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_nested_stre__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_partition_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql deleted file mode 100644 index b6c629700a04..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c___names_partition_data__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_data_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "postgres".test_normalization."nested_stream_with_c___long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql deleted file mode 100644 index 037c53598daf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c__column___with__quotes__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "postgres".test_normalization."nested_stream_with_c___long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql deleted file mode 100644 index 6d3ed538c39d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c__ion_double_array_data__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - "id", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "postgres".test_normalization."nested_stream_with_c___long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql index 0939d42acba7..8bd851a2cbb4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql @@ -3,7 +3,50 @@ create table "postgres".test_normalization."non_nested_stream_wi__lting_into_long_names__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 +), __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab1 +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 +), __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') as + varchar +)) as _airbyte_non_nested___nto_long_names_hashid, + tmp.* +from __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab2 tmp +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 +)-- Final base SQL model select "id", "date", @@ -11,7 +54,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_non_nested___nto_long_names_hashid -from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3" +from __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab3 -- non_nested_stream_wi__lting_into_long_names from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..3734105e0e64 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias.sql @@ -0,0 +1,58 @@ + + + create table "postgres".test_normalization."unnest_alias__dbt_tmp" + as ( + +with __dbt__cte__unnest_alias_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path(_airbyte_data, 'children') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 +), __dbt__cte__unnest_alias_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_ab1 +-- unnest_alias +where 1 = 1 +), __dbt__cte__unnest_alias_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(children as + varchar +), '') as + varchar +)) as _airbyte_unnest_alias_hashid, + tmp.* +from __dbt__cte__unnest_alias_ab2 tmp +-- unnest_alias +where 1 = 1 +)-- Final base SQL model +select + "id", + children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from __dbt__cte__unnest_alias_ab3 +-- unnest_alias from "postgres".test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql new file mode 100644 index 000000000000..8ddb40ab5a18 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql @@ -0,0 +1,65 @@ + + + create table "postgres".test_normalization."unnest_alias_childre__column___with__quotes__dbt_tmp" + as ( + +with __dbt__cte__unnest_alias_childre__column___with__quotes_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_owner_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias_children_owner" as table_alias +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes +cross join jsonb_array_elements( + case jsonb_typeof("column`_'with""_quotes") + when 'array' then "column`_'with""_quotes" + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and "column`_'with""_quotes" is not null +), __dbt__cte__unnest_alias_childre__column___with__quotes_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_owner_hashid, + cast(currency as + varchar +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_childre__column___with__quotes_ab1 +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes +where 1 = 1 +), __dbt__cte__unnest_alias_childre__column___with__quotes_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_owner_hashid as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') as + varchar +)) as _airbyte_column___with__quotes_hashid, + tmp.* +from __dbt__cte__unnest_alias_childre__column___with__quotes_ab2 tmp +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes +where 1 = 1 +)-- Final base SQL model +select + _airbyte_owner_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from __dbt__cte__unnest_alias_childre__column___with__quotes_ab3 +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes from "postgres".test_normalization."unnest_alias_children_owner" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql index 5248364bc8cd..3eca58b802c0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -3,7 +3,63 @@ create table "postgres".test_normalization."unnest_alias_children__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__unnest_alias_children_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'ab_id') as ab_id, + + jsonb_extract_path(_airbyte_nested_data, 'owner') + as "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children +cross join jsonb_array_elements( + case jsonb_typeof(children) + when 'array' then children + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and children is not null +), __dbt__cte__unnest_alias_children_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast("owner" as + jsonb +) as "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_children_ab1 +-- children at unnest_alias/children +where 1 = 1 +), __dbt__cte__unnest_alias_children_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as + varchar +), '') || '-' || coalesce(cast(ab_id as + varchar +), '') || '-' || coalesce(cast("owner" as + varchar +), '') as + varchar +)) as _airbyte_children_hashid, + tmp.* +from __dbt__cte__unnest_alias_children_ab2 tmp +-- children at unnest_alias/children +where 1 = 1 +)-- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, @@ -12,7 +68,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_children_hashid -from "postgres"._airbyte_test_normalization."unnest_alias_children_ab3" +from __dbt__cte__unnest_alias_children_ab3 -- children at unnest_alias/children from "postgres".test_normalization."unnest_alias" where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index b3074c01b9c9..1a5db276ee21 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -3,15 +3,62 @@ create table "postgres".test_normalization."unnest_alias_children_owner__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__unnest_alias_children_owner_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + jsonb_extract_path_text("owner", 'owner_id') as owner_id, + jsonb_extract_path("owner", 'column`_''with"_quotes') as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and "owner" is not null +), __dbt__cte__unnest_alias_children_owner_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_children_owner_ab1 +-- owner at unnest_alias/children/owner +where 1 = 1 +), __dbt__cte__unnest_alias_children_owner_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_children_hashid as + varchar +), '') || '-' || coalesce(cast(owner_id as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _airbyte_owner_hashid, + tmp.* +from __dbt__cte__unnest_alias_children_owner_ab2 tmp +-- owner at unnest_alias/children/owner +where 1 = 1 +)-- Final base SQL model select _airbyte_children_hashid, owner_id, + "column`_'with""_quotes", _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_owner_hashid -from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3" +from __dbt__cte__unnest_alias_children_owner_ab3 -- owner at unnest_alias/children/owner from "postgres".test_normalization."unnest_alias_children" where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql deleted file mode 100644 index ed8e802bc153..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create table "postgres".test_normalization_namespace."simple_stream_with_n__lting_into_long_names__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - "date", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_simple_stre__nto_long_names_hashid -from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3" --- simple_stream_with_n__lting_into_long_names from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql deleted file mode 100644 index 0d5dd2d7fcac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast("partition" as - varchar -), '') as - varchar -)) as _airbyte_nested_stre__nto_long_names_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2" tmp --- nested_stream_with_c__lting_into_long_names -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index b6d1d437f60a..d6c12c40671c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 44a922d69754..1ca0a693bc42 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index c2ab9a61b17c..471885042d69 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql index 08d114e6ce78..cd597b43a685 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql index 8179e5d3f565..45612b75e3f4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql index 91728d4b7441..c316d26be6b9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 52ad1cb15400..d6d10fc0d6c8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index 19af2a871603..55cb5c101462 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index 3c048edff728..966c6ffc0da2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index 62d7cc9c812b..265efcddba37 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 5d564afc1564..e1a73e653eb6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index 3e8ebb5d34d3..3919d47e491c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index 87f7dae0e73e..d5eae68e3ce0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index 1f254edd07e1..631991c71aab 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 8a1fb1abcdb9..6a2d47c4389d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql index c23eb81cdb1f..1ae16c9f80a1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -9,12 +9,12 @@ select _airbyte_nested_stre__nto_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, - {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_c__lting_into_long_names') }} as table_alias +from {{ ref('nested_stream_with_c__lting_into_long_names_scd') }} as table_alias -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 and {{ adapter.quote('partition') }} is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql index de62103f6c01..793cd84a5de2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -9,11 +9,11 @@ select _airbyte_nested_stre__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, - {{ adapter.quote('column`_\'with""_quotes') }}, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___long_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql index 03b4ba44eaa8..08d1fc9e476b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -10,10 +10,10 @@ select '_airbyte_nested_stre__nto_long_names_hashid', array_to_string('double_array_data'), array_to_string(adapter.quote('DATA')), - array_to_string(adapter.quote('column`_\'with""_quotes')), ]) }} as _airbyte_partition_hashid, tmp.* from {{ ref('nested_stream_with_c___long_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql index 5c39f919bd0c..b00d74d30b08 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql @@ -1,6 +1,5 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -17,4 +16,5 @@ from {{ ref('nested_stream_with_c___long_names_partition') }} as table_alias {{ cross_join_unnest('partition', adapter.quote('DATA')) }} where 1 = 1 and {{ adapter.quote('DATA') }} is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql index 670fef499d16..c9819b60f107 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql @@ -1,6 +1,5 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -14,4 +13,5 @@ select from {{ ref('nested_stream_with_c___names_partition_data_ab1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql index 962f839574c5..3b49ee8df80c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql @@ -1,6 +1,5 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -14,4 +13,5 @@ select from {{ ref('nested_stream_with_c___names_partition_data_ab2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql deleted file mode 100644 index 7445f62f1469..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('nested_stream_with_c___long_names_partition', 'partition', adapter.quote('column`_\'with""_quotes')) }} -select - _airbyte_partition_hashid, - {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_c___long_names_partition') }} as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} -where 1 = 1 -and {{ adapter.quote('column`_\'with""_quotes') }} is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql deleted file mode 100644 index 94780e7b4e93..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_c__column___with__quotes_ab1') }} --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql deleted file mode 100644 index 8bf1b1d6c988..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'currency', - ]) }} as _airbyte_column___with__quotes_hashid, - tmp.* -from {{ ref('nested_stream_with_c__column___with__quotes_ab2') }} tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql index d5254e59dd9e..4cb4339228f3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql @@ -1,6 +1,5 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -17,4 +16,5 @@ from {{ ref('nested_stream_with_c___long_names_partition') }} as table_alias {{ cross_join_unnest('partition', 'double_array_data') }} where 1 = 1 and double_array_data is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql index 0f5f7ae3a86e..c55b219147a5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql @@ -1,6 +1,5 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -14,4 +13,5 @@ select from {{ ref('nested_stream_with_c__ion_double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql index 7231c119bc08..a265778ea33e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql @@ -1,6 +1,5 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -14,4 +13,5 @@ select from {{ ref('nested_stream_with_c__ion_double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql index 6d69fcd436cf..8ed7a66e9c21 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql index a257c6a59026..7c6a65d59c40 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql index 9f67e97c0ad1..9ec56833ba85 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql index 054b91557303..ba17ef7c35c2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql index 1c7fb942b403..37016932aeff 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index fbf3b9b1c3a2..ebe6b7e37399 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -14,5 +14,4 @@ select from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index 0af0456e3965..0550f4199227 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -14,5 +14,4 @@ select from {{ ref('unnest_alias_ab1') }} -- unnest_alias where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 39fdd94517f6..33128527b34d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -14,5 +14,4 @@ select from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_childre__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_childre__column___with__quotes_ab1.sql new file mode 100644 index 000000000000..c96f0cef8c59 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_childre__column___with__quotes_ab1.sql @@ -0,0 +1,19 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +{{ unnest_cte('unnest_alias_children_owner', 'owner', adapter.quote('column`_\'with""_quotes')) }} +select + _airbyte_owner_hashid, + {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('unnest_alias_children_owner') }} as table_alias +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes +{{ cross_join_unnest('owner', adapter.quote('column`_\'with""_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column`_\'with""_quotes') }} is not null + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_childre__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_childre__column___with__quotes_ab2.sql new file mode 100644 index 000000000000..9ae3e38d07f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_childre__column___with__quotes_ab2.sql @@ -0,0 +1,16 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_owner_hashid, + cast(currency as {{ dbt_utils.type_string() }}) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('unnest_alias_childre__column___with__quotes_ab1') }} +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_childre__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_childre__column___with__quotes_ab3.sql new file mode 100644 index 000000000000..2d98283f8ae4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_childre__column___with__quotes_ab3.sql @@ -0,0 +1,16 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + '_airbyte_owner_hashid', + 'currency', + ]) }} as _airbyte_column___with__quotes_hashid, + tmp.* +from {{ ref('unnest_alias_childre__column___with__quotes_ab2') }} tmp +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 45a193100da8..e7ee275496af 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,6 +1,5 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index b93715ce6a95..e31dd1612be3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,6 +1,5 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index 5080e4179aef..435a43473909 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,6 +1,5 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 6ae9fd7cf4ac..1ff47f2c4175 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -8,6 +8,7 @@ select _airbyte_children_hashid, {{ json_extract_scalar(adapter.quote('owner'), ['owner_id'], ['owner_id']) }} as owner_id, + {{ json_extract_array(adapter.quote('owner'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index 4ca313879bc7..5ead162431b2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -8,6 +8,7 @@ select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, + {{ adapter.quote('column`_\'with""_quotes') }}, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index a25ec48b8a56..34766591d5ea 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -9,6 +9,7 @@ select {{ dbt_utils.surrogate_key([ '_airbyte_children_hashid', 'owner_id', + array_to_string(adapter.quote('column`_\'with""_quotes')), ]) }} as _airbyte_owner_hashid, tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql index b78dcd21c316..294db992bedb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization_namespace", tags = [ "top-level-intermediate" ] ) }} @@ -14,4 +14,5 @@ select from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias -- simple_stream_with_n__lting_into_long_names where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql index 14ce22350f65..6a2e84a5f65a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization_namespace", tags = [ "top-level-intermediate" ] ) }} @@ -14,4 +14,5 @@ select from {{ ref('simple_stream_with_n__lting_into_long_names_ab1') }} -- simple_stream_with_n__lting_into_long_names where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql index 9d005b773352..cb7e11e4f56e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization_namespace", tags = [ "top-level-intermediate" ] ) }} @@ -14,4 +14,5 @@ select from {{ ref('simple_stream_with_n__lting_into_long_names_ab2') }} tmp -- simple_stream_with_n__lting_into_long_names where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql index 19fc71c02c38..39e1d4360b02 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -1,7 +1,8 @@ {{ config( - indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key_scd','_airbyte_emitted_at'],'type': 'btree'}], unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", + post_hook = ['drop table if exists _airbyte_test_normalization.nested_stream_with_c__lting_into_long_names_ab3'], tags = [ "top-level" ] ) }} with @@ -63,13 +64,13 @@ scd_data as ( {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( + case when row_number() over ( partition by {{ adapter.quote('id') }} order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_stre__nto_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__ion_double_array_data.sql new file mode 100644 index 000000000000..f04c7bd5ea80 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__ion_double_array_data.sql @@ -0,0 +1,18 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + {{ adapter.quote('id') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from {{ ref('nested_stream_with_c__ion_double_array_data_ab3') }} +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_c___long_names_partition') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql index 9b80b49946e7..07afd717660d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql @@ -1,5 +1,5 @@ {{ config( - indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + indexes = [{'columns':['_airbyte_unique_key'],'unique':True}], unique_key = "_airbyte_unique_key", schema = "test_normalization", tags = [ "top-level" ] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..6ee55b4d285d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + adapter.quote('date'), + adapter.quote('partition'), + ]) }} as _airbyte_nested_stre__nto_long_names_hashid, + tmp.* +from {{ ref('nested_stream_with_c__lting_into_long_names_ab2') }} tmp +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index 2dcfc8cdeb65..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - {{ adapter.quote('id') }}, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql new file mode 100644 index 000000000000..acdd31e141be --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql @@ -0,0 +1,19 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + {{ adapter.quote('id') }}, + {{ adapter.quote('date') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_simple_stre__nto_long_names_hashid +from {{ ref('simple_stream_with_n__lting_into_long_names_ab3') }} +-- simple_stream_with_n__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index 2bbf1b892334..6dc35dcc12eb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "test_normalization", tags = [ "top-level" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 5a93704d86f2..e5c9b4cd79d6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "test_normalization", tags = [ "top-level" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql index dcf9220bfb32..7b282f49a3be 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "test_normalization", tags = [ "nested" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 54f7afc91440..bdcdbf7714d4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "test_normalization", tags = [ "nested" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 54c2c46282db..72792d9a6fca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "test_normalization", tags = [ "top-level" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql deleted file mode 100644 index e8cac7be4d93..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_nested_stre__nto_long_names_hashid, - double_array_data, - {{ adapter.quote('DATA') }}, - {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_partition_hashid -from {{ ref('nested_stream_with_c___long_names_partition_ab3') }} --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_c__lting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql deleted file mode 100644 index f254f99ad20a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_data_hashid -from {{ ref('nested_stream_with_c___names_partition_data_ab3') }} --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_c___long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql deleted file mode 100644 index 75c2b74581e6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from {{ ref('nested_stream_with_c__column___with__quotes_ab3') }} --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_c___long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql deleted file mode 100644 index 9311ea360679..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - {{ adapter.quote('id') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from {{ ref('nested_stream_with_c__ion_double_array_data_ab3') }} --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_c___long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql index cabe1843d8af..18a6ec21a3f5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "test_normalization", tags = [ "top-level" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..1a78877ff526 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql @@ -0,0 +1,18 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + {{ adapter.quote('id') }}, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql new file mode 100644 index 000000000000..fbbc9cd75783 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql @@ -0,0 +1,17 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_owner_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from {{ ref('unnest_alias_childre__column___with__quotes_ab3') }} +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes from {{ ref('unnest_alias_children_owner') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 397f6ab07fb5..e117b88908e9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,6 +1,5 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "test_normalization", tags = [ "nested" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 7aeb294ab08d..23c35a591bcb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "test_normalization", tags = [ "nested" ] ) }} @@ -8,6 +8,7 @@ select _airbyte_children_hashid, owner_id, + {{ adapter.quote('column`_\'with""_quotes') }}, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql deleted file mode 100644 index 44d939eb5a0d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization_namespace", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - {{ adapter.quote('id') }}, - {{ adapter.quote('date') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_simple_stre__nto_long_names_hashid -from {{ ref('simple_stream_with_n__lting_into_long_names_ab3') }} --- simple_stream_with_n__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql deleted file mode 100644 index b0f0198eb0e3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - adapter.quote('date'), - adapter.quote('partition'), - ]) }} as _airbyte_nested_stre__nto_long_names_hashid, - tmp.* -from {{ ref('nested_stream_with_c__lting_into_long_names_ab2') }} tmp --- nested_stream_with_c__lting_into_long_names -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index ce2e7a3ec5b5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path(_airbyte_data, 'conflict_stream_array') as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index a4df9ebe22b0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."conflict_stream_array_ab1" --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index de392f8c307b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(conflict_stream_array as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_array_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_array_ab2" tmp --- conflict_stream_array -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql deleted file mode 100644 index da03bc0bb8db..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - jsonb_extract_path_text(conflict_stream_name, 'groups') as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql deleted file mode 100644 index f40efe7753f1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as - varchar -) as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql deleted file mode 100644 index 585ae9060bfe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as - varchar -), '') || '-' || coalesce(cast(groups as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index edec407e9552..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - - jsonb_extract_path(table_alias._airbyte_data, 'conflict_stream_name') - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index fed1290ad9a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast(conflict_stream_name as - jsonb -) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."conflict_stream_name_ab1" --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index d82b4294172d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(conflict_stream_name as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_name_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_name_ab2" tmp --- conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index f0ca905a6656..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - jsonb_extract_path(table_alias.conflict_stream_name, 'conflict_stream_name') - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index a3a5870bec5d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as - jsonb -) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index 8435a2464b7b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as - varchar -), '') || '-' || coalesce(cast(conflict_stream_name as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index abb9d5899986..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'conflict_stream_scalar') as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index a5a94a9cb6c1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast(conflict_stream_scalar as - bigint -) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1" --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 66ad40a66f43..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(conflict_stream_scalar as - varchar -), '') as - varchar -)) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp --- conflict_stream_scalar -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql deleted file mode 100644 index f6e2700418b3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_stre__nto_long_names_hashid, - jsonb_extract_path("partition", 'double_array_data') as double_array_data, - jsonb_extract_path("partition", 'DATA') as "DATA", - jsonb_extract_path("partition", 'column`_''with"_quotes') as "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and "partition" is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql deleted file mode 100644 index 451bd82174e7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stre__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql deleted file mode 100644 index eb31fad896ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_nested_stre__nto_long_names_hashid as - varchar -), '') || '-' || coalesce(cast(double_array_data as - varchar -), '') || '-' || coalesce(cast("DATA" as - varchar -), '') || '-' || coalesce(cast("column`_'with""_quotes" as - varchar -), '') as - varchar -)) as _airbyte_partition_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2" tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql deleted file mode 100644 index 602dad2547f8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -cross join jsonb_array_elements( - case jsonb_typeof("DATA") - when 'array' then "DATA" - else '[]' end - ) as _airbyte_nested_data -where 1 = 1 -and "DATA" is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql deleted file mode 100644 index c51fbcb9e61f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - varchar -) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql deleted file mode 100644 index 94e1500a8681..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') as - varchar -)) as _airbyte_data_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2" tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql deleted file mode 100644 index d6e58ff5015a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -cross join jsonb_array_elements( - case jsonb_typeof("column`_'with""_quotes") - when 'array' then "column`_'with""_quotes" - else '[]' end - ) as _airbyte_nested_data -where 1 = 1 -and "column`_'with""_quotes" is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql deleted file mode 100644 index 3517ce920c9a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as - varchar -) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql deleted file mode 100644 index 37d739b12c2c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') as - varchar -)) as _airbyte_column___with__quotes_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2" tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql deleted file mode 100644 index e18ebddd1a8c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_partition_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'id') as "id", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -cross join jsonb_array_elements( - case jsonb_typeof(double_array_data) - when 'array' then double_array_data - else '[]' end - ) as _airbyte_nested_data -where 1 = 1 -and double_array_data is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql deleted file mode 100644 index d716b1eb2f5d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast("id" as - varchar -) as "id", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql deleted file mode 100644 index f4511754ecfc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as - varchar -), '') || '-' || coalesce(cast("id" as - varchar -), '') as - varchar -)) as _airbyte_double_array_data_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2" tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql deleted file mode 100644 index fd8bdeb88d0c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - - jsonb_extract_path(table_alias._airbyte_data, 'partition') - as "partition", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_c__lting_into_long_names -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql deleted file mode 100644 index 3eacd30f282f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,22 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast("date" as - varchar -) as "date", - cast("partition" as - jsonb -) as "partition", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1" --- nested_stream_with_c__lting_into_long_names -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql deleted file mode 100644 index 2a1e4ca28f20..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_wi__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql deleted file mode 100644 index 51620aee9290..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast("date" as - varchar -) as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1" --- non_nested_stream_wi__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql deleted file mode 100644 index 151cdb6e10a3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') as - varchar -)) as _airbyte_non_nested___nto_long_names_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2" tmp --- non_nested_stream_wi__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index b70f91add752..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path(_airbyte_data, 'children') as children, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index 86b9fb2c93f3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - children, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."unnest_alias_ab1" --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index f23e4c1aca53..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(children as - varchar -), '') as - varchar -)) as _airbyte_unnest_alias_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."unnest_alias_ab2" tmp --- unnest_alias -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 04ce3aa2eacc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _airbyte_unnest_alias_hashid, - jsonb_extract_path_text(_airbyte_nested_data, 'ab_id') as ab_id, - - jsonb_extract_path(_airbyte_nested_data, 'owner') - as "owner", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."unnest_alias" as table_alias --- children at unnest_alias/children -cross join jsonb_array_elements( - case jsonb_typeof(children) - when 'array' then children - else '[]' end - ) as _airbyte_nested_data -where 1 = 1 -and children is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index 00521425f33e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - bigint -) as ab_id, - cast("owner" as - jsonb -) as "owner", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."unnest_alias_children_ab1" --- children at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index f11f03175e3b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as - varchar -), '') || '-' || coalesce(cast(ab_id as - varchar -), '') || '-' || coalesce(cast("owner" as - varchar -), '') as - varchar -)) as _airbyte_children_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."unnest_alias_children_ab2" tmp --- children at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 3aafcf647131..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - jsonb_extract_path_text("owner", 'owner_id') as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization."unnest_alias_children" as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and "owner" is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index b2ca5615ca90..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - bigint -) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1" --- owner at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 8239aba10289..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_children_hashid as - varchar -), '') || '-' || coalesce(cast(owner_id as - varchar -), '') as - varchar -)) as _airbyte_owner_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp --- owner at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql deleted file mode 100644 index 5211a6eeff16..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_n__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql deleted file mode 100644 index 8f67edcf78c3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - varchar -) as "id", - cast("date" as - varchar -) as "date", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1" --- simple_stream_with_n__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql deleted file mode 100644 index 1c204c3383bb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') as - varchar -)) as _airbyte_simple_stre__nto_long_names_hashid, - tmp.* -from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2" tmp --- simple_stream_with_n__lting_into_long_names -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c___long_names_partition.sql new file mode 100644 index 000000000000..e4b16228c0ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c___long_names_partition.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."nested_stream_with_c___long_names_partition" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "nested_stream_with_c___long_names_partitio__dbt_tmp" + ); + + insert into "postgres".test_normalization."nested_stream_with_c___long_names_partition" ("_airbyte_nested_stre__nto_long_names_hashid", "double_array_data", "DATA", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid") + ( + select "_airbyte_nested_stre__nto_long_names_hashid", "double_array_data", "DATA", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid" + from "nested_stream_with_c___long_names_partitio__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c___names_partition_data.sql new file mode 100644 index 000000000000..5c12b7ee08da --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c___names_partition_data.sql @@ -0,0 +1,9 @@ + + + + insert into "postgres".test_normalization."nested_stream_with_c___names_partition_data" ("_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid") + ( + select "_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid" + from "nested_stream_with_c___names_partition_dat__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__ion_double_array_data.sql new file mode 100644 index 000000000000..8b12de7eda4a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__ion_double_array_data.sql @@ -0,0 +1,9 @@ + + + + insert into "postgres".test_normalization."nested_stream_with_c__ion_double_array_data" ("_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid") + ( + select "_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid" + from "nested_stream_with_c__ion_double_array_dat__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..a66bd29ad0a4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql @@ -0,0 +1,60 @@ + + + + create table "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3" + as ( + +with __dbt__cte__nested_stream_with_c__lting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + + jsonb_extract_path(table_alias._airbyte_data, 'partition') + as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + +), __dbt__cte__nested_stream_with_c__lting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + cast("partition" as + jsonb +) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_c__lting_into_long_names_ab1 +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast("partition" as + varchar +), '') as + varchar +)) as _airbyte_nested_stre__nto_long_names_hashid, + tmp.* +from __dbt__cte__nested_stream_with_c__lting_into_long_names_ab2 tmp +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index 7199e9f5f914..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "postgres".test_normalization."unnest_alias" - where (_airbyte_ab_id) in ( - select (_airbyte_ab_id) - from "unnest_alias__dbt_tmp" - ); - - insert into "postgres".test_normalization."unnest_alias" ("id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid") - ( - select "id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid" - from "unnest_alias__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql new file mode 100644 index 000000000000..7fbbe6459fa0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization_namespace."simple_stream_with_n__lting_into_long_names" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "simple_stream_with_n__lting_into_long_name__dbt_tmp" + ); + + insert into "postgres".test_normalization_namespace."simple_stream_with_n__lting_into_long_names" ("id", "date", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_simple_stre__nto_long_names_hashid") + ( + select "id", "date", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_simple_stre__nto_long_names_hashid" + from "simple_stream_with_n__lting_into_long_name__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql index 24d04f946c89..ab5935b1d289 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -3,7 +3,48 @@ create table "postgres".test_normalization."conflict_stream_array__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__conflict_stream_array_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path(_airbyte_data, 'conflict_stream_array') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 +), __dbt__cte__conflict_stream_array_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_array_ab1 +-- conflict_stream_array +where 1 = 1 +), __dbt__cte__conflict_stream_array_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_array as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from __dbt__cte__conflict_stream_array_ab2 tmp +-- conflict_stream_array +where 1 = 1 +)-- Final base SQL model select "id", conflict_stream_array, @@ -11,7 +52,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_array_ab3" +from __dbt__cte__conflict_stream_array_ab3 -- conflict_stream_array from "postgres".test_normalization._airbyte_raw_conflict_stream_array where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql index a664bb56d781..8b379cafff07 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -3,7 +3,52 @@ create table "postgres".test_normalization."conflict_stream_name__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + + jsonb_extract_path(table_alias._airbyte_data, 'conflict_stream_name') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast(conflict_stream_name as + jsonb +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_ab1 +-- conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_name as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_ab2 tmp +-- conflict_stream_name +where 1 = 1 +)-- Final base SQL model select "id", conflict_stream_name, @@ -11,7 +56,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_name_ab3" +from __dbt__cte__conflict_stream_name_ab3 -- conflict_stream_name from "postgres".test_normalization._airbyte_raw_conflict_stream_name where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql index 0e06a3243fa2..1729627b03eb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql @@ -3,7 +3,49 @@ create table "postgres".test_normalization."conflict_stream_name___conflict_stream_name__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__conflict_stream_name___conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + jsonb_extract_path_text(conflict_stream_name, 'groups') as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null +), __dbt__cte__conflict_stream_name___conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as + varchar +) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name___conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name___conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + varchar +), '') || '-' || coalesce(cast(groups as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from __dbt__cte__conflict_stream_name___conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +)-- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, groups, @@ -11,7 +53,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3" +from __dbt__cte__conflict_stream_name___conflict_stream_name_ab3 -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index df6e327e1620..dc9657144317 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -3,7 +3,51 @@ create table "postgres".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + jsonb_extract_path(table_alias.conflict_stream_name, 'conflict_stream_name') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null +), __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as + jsonb +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as + varchar +), '') || '-' || coalesce(cast(conflict_stream_name as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +)-- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, @@ -11,7 +55,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 -- conflict_stream_name at conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name" where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql index bdcd7a2664c1..2e06b81d6a56 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -3,7 +3,50 @@ create table "postgres".test_normalization."conflict_stream_scalar__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__conflict_stream_scalar_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'conflict_stream_scalar') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 +), __dbt__cte__conflict_stream_scalar_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_scalar_ab1 +-- conflict_stream_scalar +where 1 = 1 +), __dbt__cte__conflict_stream_scalar_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_scalar as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from __dbt__cte__conflict_stream_scalar_ab2 tmp +-- conflict_stream_scalar +where 1 = 1 +)-- Final base SQL model select "id", conflict_stream_scalar, @@ -11,7 +54,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid -from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3" +from __dbt__cte__conflict_stream_scalar_ab3 -- conflict_stream_scalar from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql deleted file mode 100644 index a9781fcf92d2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c___long_names_partition__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_nested_stre__nto_long_names_hashid, - double_array_data, - "DATA", - "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_partition_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3" --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql deleted file mode 100644 index b6c629700a04..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c___names_partition_data__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_data_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "postgres".test_normalization."nested_stream_with_c___long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql deleted file mode 100644 index 037c53598daf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c__column___with__quotes__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3" --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "postgres".test_normalization."nested_stream_with_c___long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql deleted file mode 100644 index 6d3ed538c39d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c__ion_double_array_data__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_partition_hashid, - "id", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3" --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "postgres".test_normalization."nested_stream_with_c___long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql index 0939d42acba7..8bd851a2cbb4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql @@ -3,7 +3,50 @@ create table "postgres".test_normalization."non_nested_stream_wi__lting_into_long_names__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 +), __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab1 +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 +), __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') as + varchar +)) as _airbyte_non_nested___nto_long_names_hashid, + tmp.* +from __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab2 tmp +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 +)-- Final base SQL model select "id", "date", @@ -11,7 +54,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_non_nested___nto_long_names_hashid -from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3" +from __dbt__cte__non_nested_stream_wi__lting_into_long_names_ab3 -- non_nested_stream_wi__lting_into_long_names from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..3734105e0e64 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias.sql @@ -0,0 +1,58 @@ + + + create table "postgres".test_normalization."unnest_alias__dbt_tmp" + as ( + +with __dbt__cte__unnest_alias_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path(_airbyte_data, 'children') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 +), __dbt__cte__unnest_alias_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_ab1 +-- unnest_alias +where 1 = 1 +), __dbt__cte__unnest_alias_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(children as + varchar +), '') as + varchar +)) as _airbyte_unnest_alias_hashid, + tmp.* +from __dbt__cte__unnest_alias_ab2 tmp +-- unnest_alias +where 1 = 1 +)-- Final base SQL model +select + "id", + children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from __dbt__cte__unnest_alias_ab3 +-- unnest_alias from "postgres".test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql new file mode 100644 index 000000000000..8ddb40ab5a18 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql @@ -0,0 +1,65 @@ + + + create table "postgres".test_normalization."unnest_alias_childre__column___with__quotes__dbt_tmp" + as ( + +with __dbt__cte__unnest_alias_childre__column___with__quotes_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_owner_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias_children_owner" as table_alias +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes +cross join jsonb_array_elements( + case jsonb_typeof("column`_'with""_quotes") + when 'array' then "column`_'with""_quotes" + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and "column`_'with""_quotes" is not null +), __dbt__cte__unnest_alias_childre__column___with__quotes_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_owner_hashid, + cast(currency as + varchar +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_childre__column___with__quotes_ab1 +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes +where 1 = 1 +), __dbt__cte__unnest_alias_childre__column___with__quotes_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_owner_hashid as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') as + varchar +)) as _airbyte_column___with__quotes_hashid, + tmp.* +from __dbt__cte__unnest_alias_childre__column___with__quotes_ab2 tmp +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes +where 1 = 1 +)-- Final base SQL model +select + _airbyte_owner_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from __dbt__cte__unnest_alias_childre__column___with__quotes_ab3 +-- column___with__quotes at unnest_alias/children/owner/column`_'with"_quotes from "postgres".test_normalization."unnest_alias_children_owner" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql index 5248364bc8cd..3eca58b802c0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -3,7 +3,63 @@ create table "postgres".test_normalization."unnest_alias_children__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__unnest_alias_children_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'ab_id') as ab_id, + + jsonb_extract_path(_airbyte_nested_data, 'owner') + as "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children +cross join jsonb_array_elements( + case jsonb_typeof(children) + when 'array' then children + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and children is not null +), __dbt__cte__unnest_alias_children_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast("owner" as + jsonb +) as "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_children_ab1 +-- children at unnest_alias/children +where 1 = 1 +), __dbt__cte__unnest_alias_children_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as + varchar +), '') || '-' || coalesce(cast(ab_id as + varchar +), '') || '-' || coalesce(cast("owner" as + varchar +), '') as + varchar +)) as _airbyte_children_hashid, + tmp.* +from __dbt__cte__unnest_alias_children_ab2 tmp +-- children at unnest_alias/children +where 1 = 1 +)-- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, @@ -12,7 +68,7 @@ select _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_children_hashid -from "postgres"._airbyte_test_normalization."unnest_alias_children_ab3" +from __dbt__cte__unnest_alias_children_ab3 -- children at unnest_alias/children from "postgres".test_normalization."unnest_alias" where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index b3074c01b9c9..1a5db276ee21 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -3,15 +3,62 @@ create table "postgres".test_normalization."unnest_alias_children_owner__dbt_tmp" as ( --- Final base SQL model +with __dbt__cte__unnest_alias_children_owner_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + jsonb_extract_path_text("owner", 'owner_id') as owner_id, + jsonb_extract_path("owner", 'column`_''with"_quotes') as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and "owner" is not null +), __dbt__cte__unnest_alias_children_owner_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_children_owner_ab1 +-- owner at unnest_alias/children/owner +where 1 = 1 +), __dbt__cte__unnest_alias_children_owner_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_children_hashid as + varchar +), '') || '-' || coalesce(cast(owner_id as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _airbyte_owner_hashid, + tmp.* +from __dbt__cte__unnest_alias_children_owner_ab2 tmp +-- owner at unnest_alias/children/owner +where 1 = 1 +)-- Final base SQL model select _airbyte_children_hashid, owner_id, + "column`_'with""_quotes", _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at, _airbyte_owner_hashid -from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3" +from __dbt__cte__unnest_alias_children_owner_ab3 -- owner at unnest_alias/children/owner from "postgres".test_normalization."unnest_alias_children" where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql deleted file mode 100644 index ed8e802bc153..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - create table "postgres".test_normalization_namespace."simple_stream_with_n__lting_into_long_names__dbt_tmp" - as ( - --- Final base SQL model -select - "id", - "date", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_simple_stre__nto_long_names_hashid -from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3" --- simple_stream_with_n__lting_into_long_names from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql deleted file mode 100644 index 0d5dd2d7fcac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql +++ /dev/null @@ -1,20 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast("partition" as - varchar -), '') as - varchar -)) as _airbyte_nested_stre__nto_long_names_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2" tmp --- nested_stream_with_c__lting_into_long_names -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml new file mode 100755 index 000000000000..522b1e595e8a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["modified_models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml new file mode 100644 index 000000000000..9ad815875900 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 6e1c220c3cbc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'name') as "name", - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 88c28f9fa253..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast("name" as - varchar -) as "name", - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1" --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index ed7c144f7d77..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'currency') as currency, - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, - jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", - jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, - jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, - jsonb_extract_path_text(_airbyte_data, 'USD') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index 7e00f6385af7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,37 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast(currency as - varchar -) as currency, - cast(nullif("date", '') as - date -) as "date", - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - varchar -) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1" --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 6e6073e308e0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'currency') as currency, - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, - jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", - jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, - jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, - jsonb_extract_path_text(_airbyte_data, 'USD') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 6cbcedd760a2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,37 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast(currency as - varchar -) as currency, - cast(nullif("date", '') as - date -) as "date", - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - varchar -) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."exchange_rate_ab1" --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index e2b7bfd34d06..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast(timestamp_col as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(hkd_special___characters as - varchar -), '') || '-' || coalesce(cast(nzd as - varchar -), '') || '-' || coalesce(cast(usd as - varchar -), '') as - varchar -)) as _airbyte_exchange_rate_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."exchange_rate_ab2" tmp --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 7c1fca4961ff..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'name') as "name", - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index a9e2b29e6210..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast("name" as - varchar -) as "name", - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1" --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 4380fdbedd6b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 7371209e09a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index 0f6ee557356e..b3241957f634 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -33,13 +33,13 @@ scd_data as ( _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( + case when row_number() over ( partition by "id" order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_cdc_excluded_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 2af61c0cc3b8..ec0976fde78e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,7 +1,7 @@ - create table "postgres"."test_normalization"."dedup_exchange_rate_scd__dbt_tmp" + create table "postgres".test_normalization."dedup_exchange_rate_scd" as ( with diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..831036d8fa21 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,99 @@ + + + + create table "postgres".test_normalization."pos_dedup_cdcx_scd" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when row_number() over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + varchar +), cast(_ab_cdc_updated_at as + varchar +), cast(_ab_cdc_log_pos as + varchar +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_log_pos as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 68c9ff694078..cf39240190db 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -29,13 +29,13 @@ scd_data as ( _airbyte_emitted_at desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( + case when row_number() over ( partition by "id" order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_renamed_dedup_cdc_excluded_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..43de64e45947 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,70 @@ + + + + create table "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3" + as ( + +with __dbt__cte__dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__dedup_cdc_excluded_ab1 +-- dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') as + varchar +)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__dedup_cdc_excluded_ab2 tmp +-- dedup_cdc_excluded +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..4fea55f8e036 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,88 @@ + + + + create table "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3" + as ( + +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast(currency as + varchar +) as currency, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + varchar +) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(hkd_special___characters as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from __dbt__cte__dedup_exchange_rate_ab2 tmp +-- dedup_exchange_rate +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 2c7e88caddef..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - - create table "postgres".test_normalization."exchange_rate" - as ( - --- Final base SQL model -select - "id", - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from "postgres"._airbyte_test_normalization."exchange_rate_ab3" --- exchange_rate from "postgres".test_normalization._airbyte_raw_exchange_rate -where 1 = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..afe376e5a288 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,26 @@ + + + + create table "postgres".test_normalization."pos_dedup_cdcx" + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from "postgres".test_normalization."pos_dedup_cdcx_scd" +-- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..fdb312f61494 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,76 @@ + + + + create table "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + as ( + +with __dbt__cte__pos_dedup_cdcx_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + +), __dbt__cte__pos_dedup_cdcx_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__pos_dedup_cdcx_ab1 +-- pos_dedup_cdcx +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_log_pos as + varchar +), '') as + varchar +)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from __dbt__cte__pos_dedup_cdcx_ab2 tmp +-- pos_dedup_cdcx +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..9c0cb18e8092 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,46 @@ + + + + create table "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" + as ( + +with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__renamed_dedup_cdc_excluded_ab1 +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 48e2e8023174..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,97 +0,0 @@ - - - create table "postgres".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - as ( - -with - -input_data as ( - select * - from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" - -- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(coalesce(cast("id" as - varchar -), '') as - varchar -)) as _airbyte_unique_key, - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by "id" - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by "id" - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as - varchar -), cast(_ab_cdc_updated_at as - varchar -), cast(_ab_cdc_log_pos as - varchar -) - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(coalesce(cast(_airbyte_unique_key as - varchar -), '') || '-' || coalesce(cast(_airbyte_start_at as - varchar -), '') || '-' || coalesce(cast(_airbyte_emitted_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_updated_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_log_pos as - varchar -), '') as - varchar -)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..93ccc8936769 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,109 @@ + + + create table "postgres".test_normalization."exchange_rate__dbt_tmp" + as ( + +with __dbt__cte__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + jsonb_extract_path_text(_airbyte_data, 'column`_''with"_quotes') as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast(currency as + varchar +) as currency, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + varchar +) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast("column`_'with""_quotes" as + varchar +) as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(hkd_special___characters as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__cte__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + "id", + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__cte__exchange_rate_ab3 +-- exchange_rate from "postgres".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 25e9cb949467..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create table "postgres".test_normalization."pos_dedup_cdcx__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_unique_key, - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from "postgres".test_normalization."pos_dedup_cdcx_scd" --- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 63d83886e5f1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("name" as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_lsn as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_updated_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as - varchar -), '') as - varchar -)) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 162849517206..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast(timestamp_col as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(hkd_special___characters as - varchar -), '') || '-' || coalesce(cast(nzd as - varchar -), '') || '-' || coalesce(cast(usd as - varchar -), '') as - varchar -)) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index fceb899227d1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,25 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("name" as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_lsn as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_updated_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_log_pos as - varchar -), '') as - varchar -)) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 732f2bd1a1ea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') as - varchar -)) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 6ff5c5605fb8..575e42512c15 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index a90b1c1e09af..3b317e6ef438 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 5575218fafd2..cbe16752071c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 74859161cd8f..68d124105712 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index e0f71f0e7cf2..18ee0a6e8f62 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -14,11 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, + {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 56f31c5d75f3..f450cd8c6d56 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -14,11 +14,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, + cast({{ adapter.quote('column`_\'with""_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column`_\'with""_quotes') }}, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 8784ce3bc62d..dec14fefa4de 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -15,10 +15,10 @@ select 'hkd_special___characters', 'nzd', 'usd', + adapter.quote('column`_\'with""_quotes'), ]) }} as _airbyte_exchange_rate_hashid, tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index fb9d0b0df354..02a4a88e458e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -18,4 +18,5 @@ select from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index adf58230d903..a4dc90d99c53 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -18,4 +18,5 @@ select from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql index ca844abec811..4ff1ba687b9b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql index e2b60bbe8a71..c1b2178b8717 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -1,6 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index 4e4500b5ad39..0a549bc1071c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -1,7 +1,8 @@ {{ config( - indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key_scd','_airbyte_emitted_at'],'type': 'btree'}], unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", + post_hook = ['drop table if exists _airbyte_test_normalization.dedup_cdc_excluded_ab3'], tags = [ "top-level" ] ) }} with @@ -65,13 +66,13 @@ scd_data as ( _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( + case when row_number() over ( partition by {{ adapter.quote('id') }} order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_cdc_excluded_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 44047d79d0cd..27bd81359b44 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,7 +1,8 @@ {{ config( - indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key_scd','_airbyte_emitted_at'],'type': 'btree'}], unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", + post_hook = ['drop table if exists _airbyte_test_normalization.dedup_exchange_rate_ab3'], tags = [ "top-level" ] ) }} with @@ -72,13 +73,13 @@ scd_data as ( {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( + case when row_number() over ( partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..702f93bb8c58 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,115 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key_scd','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + post_hook = ['drop table if exists _airbyte_test_normalization.pos_dedup_cdcx_ab3'], + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when row_number() over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 8c57c2ee55b5..a49f8b4bd019 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -1,7 +1,8 @@ {{ config( - indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key_scd','_airbyte_emitted_at'],'type': 'btree'}], unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", + post_hook = ['drop table if exists _airbyte_test_normalization.renamed_dedup_cdc_excluded_ab3'], tags = [ "top-level" ] ) }} with @@ -61,13 +62,13 @@ scd_data as ( _airbyte_emitted_at desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( + case when row_number() over ( partition by {{ adapter.quote('id') }} order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_renamed_dedup_cdc_excluded_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index 18247c346368..9ed1c0e22f2b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -1,5 +1,5 @@ {{ config( - indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + indexes = [{'columns':['_airbyte_unique_key'],'unique':True}], unique_key = "_airbyte_unique_key", schema = "test_normalization", tags = [ "top-level" ] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..8d0677ad7bec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,21 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + adapter.quote('name'), + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + ]) }} as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('dedup_cdc_excluded_ab2') }} tmp +-- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 370919ff628e..8390de3ac6d8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,5 +1,5 @@ {{ config( - indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + indexes = [{'columns':['_airbyte_unique_key'],'unique':True}], unique_key = "_airbyte_unique_key", schema = "test_normalization", tags = [ "top-level" ] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..413c3121df7b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'currency', + adapter.quote('date'), + 'timestamp_col', + adapter.quote('HKD@spéçiäl & characters'), + 'hkd_special___characters', + 'nzd', + 'usd', + ]) }} as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index b8b52c6d4d88..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - {{ adapter.quote('id') }}, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..be10bbda5ef6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,25 @@ +{{ config( + indexes = [{'columns':['_airbyte_unique_key'],'unique':True}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from {{ ref('pos_dedup_cdcx_scd') }} +-- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..030a684ac2a6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,22 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + adapter.quote('name'), + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + '_ab_cdc_log_pos', + ]) }} as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from {{ ref('pos_dedup_cdcx_ab2') }} tmp +-- pos_dedup_cdcx +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index 4bf1ea58e96e..61f1ba579eda 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -1,5 +1,5 @@ {{ config( - indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + indexes = [{'columns':['_airbyte_unique_key'],'unique':True}], unique_key = "_airbyte_unique_key", schema = "test_normalization", tags = [ "top-level" ] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..1f411cbcd20d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,17 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index afc9f924fa86..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,114 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by {{ adapter.quote('id') }} - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by {{ adapter.quote('id') }} - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..21086fc548d0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,25 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + {{ adapter.quote('id') }}, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 04c21e5fc3be..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_scd') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where 1 = 1 -and _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index e539d1844de2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - adapter.quote('name'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index ee7631c2c6ea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - 'currency', - adapter.quote('date'), - 'timestamp_col', - adapter.quote('HKD@spéçiäl & characters'), - 'hkd_special___characters', - 'nzd', - 'usd', - ]) }} as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from {{ ref('dedup_exchange_rate_ab2') }} tmp --- dedup_exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 9da3c6fea93a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - adapter.quote('name'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index b152709f2db5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..eb052e88ec5a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, + {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, + {{ json_extract_scalar('_airbyte_data', ['new_column'], ['new_column']) }} as new_column, + {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, + {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, + {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as {{ adapter.quote('HKD@spéçiäl & characters') }}, + {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, + {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..753be4337376 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast({{ adapter.quote('id') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('id') }}, + cast(currency as {{ dbt_utils.type_string() }}) as currency, + cast(new_column as {{ dbt_utils.type_float() }}) as new_column, + cast({{ empty_string_to_null(adapter.quote('date')) }} as {{ type_date() }}) as {{ adapter.quote('date') }}, + cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col, + cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }}, + cast(nzd as {{ dbt_utils.type_float() }}) as nzd, + cast(usd as {{ dbt_utils.type_bigint() }}) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('dedup_exchange_rate_ab1') }} +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..f91ebab3ecb3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, + {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, + {{ json_extract_scalar('_airbyte_data', ['new_column'], ['new_column']) }} as new_column, + {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, + {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, + {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as {{ adapter.quote('HKD@spéçiäl & characters') }}, + {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, + {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, + {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias +-- exchange_rate +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..0f09f3101177 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast({{ adapter.quote('id') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('id') }}, + cast(currency as {{ dbt_utils.type_string() }}) as currency, + cast(new_column as {{ dbt_utils.type_float() }}) as new_column, + cast({{ empty_string_to_null(adapter.quote('date')) }} as {{ type_date() }}) as {{ adapter.quote('date') }}, + cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col, + cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }}, + cast(nzd as {{ dbt_utils.type_float() }}) as nzd, + cast(usd as {{ dbt_utils.type_float() }}) as usd, + cast({{ adapter.quote('column`_\'with""_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('exchange_rate_ab1') }} +-- exchange_rate +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..f1547c8b5264 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'currency', + 'new_column', + adapter.quote('date'), + 'timestamp_col', + adapter.quote('HKD@spéçiäl & characters'), + 'nzd', + 'usd', + adapter.quote('column`_\'with""_quotes'), + ]) }} as _airbyte_exchange_rate_hashid, + tmp.* +from {{ ref('exchange_rate_ab2') }} tmp +-- exchange_rate +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..94e3f126c1d1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,21 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, + {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as {{ adapter.quote('name') }}, + {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, + {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, + {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..689a8fb307d1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,21 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, + cast({{ adapter.quote('name') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('name') }}, + cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..9422dfc609bc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,123 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key_scd','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + post_hook = ['drop table if exists _airbyte_test_normalization.dedup_exchange_rate_ab3'], + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + currency, + new_column, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + nzd, + usd, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}), currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when row_number() over ( + partition by cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}), currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) = 1 then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + currency, + new_column, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..1a9aa39a1178 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,113 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key_scd','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + post_hook = ['drop table if exists _airbyte_test_normalization.renamed_dedup_cdc_excluded_ab3'], + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when row_number() over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) = 1 and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..f3f7a253e74c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,27 @@ +{{ config( + indexes = [{'columns':['_airbyte_unique_key'],'unique':True}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + {{ adapter.quote('id') }}, + currency, + new_column, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from {{ ref('dedup_exchange_rate_scd') }} +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..b80daaa3a065 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'currency', + 'new_column', + adapter.quote('date'), + 'timestamp_col', + adapter.quote('HKD@spéçiäl & characters'), + 'nzd', + 'usd', + ]) }} as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..b6b676a94c88 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_unique_key'],'unique':True}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..be8cc745e66e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,21 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + adapter.quote('name'), + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..202af269c599 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,25 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + {{ adapter.quote('id') }}, + currency, + new_column, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + nzd, + usd, + {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/sources.yml new file mode 100644 index 000000000000..dd538a80131a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/sources.yml @@ -0,0 +1,11 @@ +version: 2 +sources: +- name: test_normalization + quoting: + database: true + schema: false + identifier: false + tables: + - name: _airbyte_raw_dedup_exchange_rate + - name: _airbyte_raw_exchange_rate + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 6e1c220c3cbc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'name') as "name", - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 88c28f9fa253..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast("name" as - varchar -) as "name", - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1" --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index ed7c144f7d77..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'currency') as currency, - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, - jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", - jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, - jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, - jsonb_extract_path_text(_airbyte_data, 'USD') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index 7e00f6385af7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,37 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast(currency as - varchar -) as currency, - cast(nullif("date", '') as - date -) as "date", - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - varchar -) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1" --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 6e6073e308e0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'currency') as currency, - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, - jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", - jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, - jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, - jsonb_extract_path_text(_airbyte_data, 'USD') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 6cbcedd760a2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,37 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast(currency as - varchar -) as currency, - cast(nullif("date", '') as - date -) as "date", - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(hkd_special___characters as - varchar -) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."exchange_rate_ab1" --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index e2b7bfd34d06..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast(timestamp_col as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(hkd_special___characters as - varchar -), '') || '-' || coalesce(cast(nzd as - varchar -), '') || '-' || coalesce(cast(usd as - varchar -), '') as - varchar -)) as _airbyte_exchange_rate_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."exchange_rate_ab2" tmp --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 7c1fca4961ff..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'name') as "name", - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index a9e2b29e6210..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast("name" as - varchar -) as "name", - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1" --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 4380fdbedd6b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index 7371209e09a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..ebe6ae45a576 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."pos_dedup_cdcx_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "pos_dedup_cdcx_scd__dbt_tmp" + ); + + insert into "postgres".test_normalization."pos_dedup_cdcx_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_ab_cdc_log_pos", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_pos_dedup_cdcx_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_ab_cdc_log_pos", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_pos_dedup_cdcx_hashid" + from "pos_dedup_cdcx_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..43de64e45947 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,70 @@ + + + + create table "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3" + as ( + +with __dbt__cte__dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__dedup_cdc_excluded_ab1 +-- dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') as + varchar +)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__dedup_cdc_excluded_ab2 tmp +-- dedup_cdc_excluded +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..4fea55f8e036 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,88 @@ + + + + create table "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3" + as ( + +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast(currency as + varchar +) as currency, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + varchar +) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(hkd_special___characters as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from __dbt__cte__dedup_exchange_rate_ab2 tmp +-- dedup_exchange_rate +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 4e20aa65740e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "postgres".test_normalization."exchange_rate" - where (_airbyte_ab_id) in ( - select (_airbyte_ab_id) - from "exchange_rate__dbt_tmp" - ); - - insert into "postgres".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") - ( - select "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid" - from "exchange_rate__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..6da0cd1efa17 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."pos_dedup_cdcx" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "pos_dedup_cdcx__dbt_tmp" + ); + + insert into "postgres".test_normalization."pos_dedup_cdcx" ("_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_ab_cdc_log_pos", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_pos_dedup_cdcx_hashid") + ( + select "_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_ab_cdc_log_pos", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_pos_dedup_cdcx_hashid" + from "pos_dedup_cdcx__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..fdb312f61494 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,76 @@ + + + + create table "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + as ( + +with __dbt__cte__pos_dedup_cdcx_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + +), __dbt__cte__pos_dedup_cdcx_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__pos_dedup_cdcx_ab1 +-- pos_dedup_cdcx +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_log_pos as + varchar +), '') as + varchar +)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from __dbt__cte__pos_dedup_cdcx_ab2 tmp +-- pos_dedup_cdcx +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..9c0cb18e8092 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,46 @@ + + + + create table "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" + as ( + +with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__renamed_dedup_cdc_excluded_ab1 +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 48e2e8023174..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,97 +0,0 @@ - - - create table "postgres".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - as ( - -with - -input_data as ( - select * - from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" - -- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(coalesce(cast("id" as - varchar -), '') as - varchar -)) as _airbyte_unique_key, - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by "id" - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by "id" - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as - varchar -), cast(_ab_cdc_updated_at as - varchar -), cast(_ab_cdc_log_pos as - varchar -) - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(coalesce(cast(_airbyte_unique_key as - varchar -), '') || '-' || coalesce(cast(_airbyte_start_at as - varchar -), '') || '-' || coalesce(cast(_airbyte_emitted_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_updated_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_log_pos as - varchar -), '') as - varchar -)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..93ccc8936769 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,109 @@ + + + create table "postgres".test_normalization."exchange_rate__dbt_tmp" + as ( + +with __dbt__cte__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + jsonb_extract_path_text(_airbyte_data, 'column`_''with"_quotes') as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast(currency as + varchar +) as currency, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + varchar +) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast("column`_'with""_quotes" as + varchar +) as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(hkd_special___characters as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__cte__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + "id", + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__cte__exchange_rate_ab3 +-- exchange_rate from "postgres".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 25e9cb949467..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,23 +0,0 @@ - - - create table "postgres".test_normalization."pos_dedup_cdcx__dbt_tmp" - as ( - --- Final base SQL model -select - _airbyte_unique_key, - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from "postgres".test_normalization."pos_dedup_cdcx_scd" --- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 63d83886e5f1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("name" as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_lsn as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_updated_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as - varchar -), '') as - varchar -)) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp --- dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 162849517206..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast(timestamp_col as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(hkd_special___characters as - varchar -), '') || '-' || coalesce(cast(nzd as - varchar -), '') || '-' || coalesce(cast(usd as - varchar -), '') as - varchar -)) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index fceb899227d1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,25 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("name" as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_lsn as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_updated_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_log_pos as - varchar -), '') as - varchar -)) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp --- pos_dedup_cdcx -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 732f2bd1a1ea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') as - varchar -)) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index 5fc6f0070232..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'currency') as currency, - jsonb_extract_path_text(_airbyte_data, 'new_column') as new_column, - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, - jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", - jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, - jsonb_extract_path_text(_airbyte_data, 'USD') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index 1f848d1a326a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,37 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - float -) as "id", - cast(currency as - varchar -) as currency, - cast(new_column as - float -) as new_column, - cast(nullif("date", '') as - date -) as "date", - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(nzd as - float -) as nzd, - cast(usd as - bigint -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1" --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 53eb0e732149..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'currency') as currency, - jsonb_extract_path_text(_airbyte_data, 'new_column') as new_column, - jsonb_extract_path_text(_airbyte_data, 'date') as "date", - jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, - jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", - jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, - jsonb_extract_path_text(_airbyte_data, 'USD') as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 6b8c0fa4997e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,37 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - float -) as "id", - cast(currency as - varchar -) as currency, - cast(new_column as - float -) as new_column, - cast(nullif("date", '') as - date -) as "date", - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."exchange_rate_ab1" --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index da83934e7c93..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') || '-' || coalesce(cast(new_column as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast(timestamp_col as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(nzd as - varchar -), '') || '-' || coalesce(cast(usd as - varchar -), '') as - varchar -)) as _airbyte_exchange_rate_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."exchange_rate_ab2" tmp --- exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index b3923b59c83c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - jsonb_extract_path_text(_airbyte_data, 'id') as "id", - jsonb_extract_path_text(_airbyte_data, 'name') as "name", - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, - jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index aa3cc798fcf9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast("id" as - bigint -) as "id", - cast("name" as - varchar -) as "name", - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - now() as _airbyte_normalized_at -from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..eca28d965b98 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,88 @@ + + + + create table "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3" + as ( + +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'new_column') as new_column, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + float +) as "id", + cast(currency as + varchar +) as currency, + cast(new_column as + float +) as new_column, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(nzd as + float +) as nzd, + cast(usd as + bigint +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast(new_column as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from __dbt__cte__dedup_exchange_rate_ab2 tmp +-- dedup_exchange_rate +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 381151b8af8c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "postgres".test_normalization."exchange_rate" - where (_airbyte_ab_id) in ( - select (_airbyte_ab_id) - from "exchange_rate__dbt_tmp" - ); - - insert into "postgres".test_normalization."exchange_rate" ("currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id") - ( - select "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id" - from "exchange_rate__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..361fd2649d9a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,70 @@ + + + + create table "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" + as ( + +with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__renamed_dedup_cdc_excluded_ab1 +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') as + varchar +)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..e10504a47e13 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,109 @@ + + + create table "postgres".test_normalization."exchange_rate__dbt_tmp" + as ( + +with __dbt__cte__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'new_column') as new_column, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + jsonb_extract_path_text(_airbyte_data, 'column`_''with"_quotes') as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + float +) as "id", + cast(currency as + varchar +) as currency, + cast(new_column as + float +) as new_column, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast("column`_'with""_quotes" as + varchar +) as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast(new_column as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__cte__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + "id", + currency, + new_column, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + nzd, + usd, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__cte__exchange_rate_ab3 +-- exchange_rate from "postgres".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql deleted file mode 100644 index 21806fa208d3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast(currency as - varchar -), '') || '-' || coalesce(cast(new_column as - varchar -), '') || '-' || coalesce(cast("date" as - varchar -), '') || '-' || coalesce(cast(timestamp_col as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(nzd as - varchar -), '') || '-' || coalesce(cast(usd as - varchar -), '') as - varchar -)) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp --- dedup_exchange_rate -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index b7b5f10b73a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast("id" as - varchar -), '') || '-' || coalesce(cast("name" as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_lsn as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_updated_at as - varchar -), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as - varchar -), '') as - varchar -)) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml new file mode 100755 index 000000000000..9ad815875900 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index c9bc7641794f..c2a9f41a3840 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -5,7 +5,7 @@ "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names_scd__dbt_tmp" - compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + compound sortkey(_airbyte_active_row,_airbyte_unique_key_scd,_airbyte_emitted_at) as ( with @@ -31,13 +31,13 @@ scd_data as ( date desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(date) over ( + case when row_number() over ( partition by id order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..4c447f2efab2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,64 @@ + + + + create table + "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names_partition__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + json_extract_path_text("partition", 'double_array_data', true) as double_array_data, + json_extract_path_text("partition", 'DATA', true) as data, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + data, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as varchar), '') || '-' || coalesce(cast(double_array_data as varchar), '') || '-' || coalesce(cast(data as varchar), '') as varchar)) as _airbyte_partition_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + data, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_partition_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql new file mode 100644 index 000000000000..d55aaf409ab3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -0,0 +1,106 @@ + + + + create table + "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names_partition_data__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text(data, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.data + where numbers.generated_number <= json_array_length(data, true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and data is not null + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as varchar) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 tmp +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_data_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..83a8a0653cb0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,106 @@ + + + + create table + "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text(double_array_data, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.double_array_data + where numbers.generated_number <= json_array_length(double_array_data, true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'id', true) end as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as varchar) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(id as varchar), '') as varchar)) as _airbyte_double_array_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +)-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index e6dd29796238..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,62 +0,0 @@ - - - - create table - "integrationtests"."test_normalization"."unnest_alias__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__unnest_alias_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - json_extract_path_text(_airbyte_data, 'children', true) as children, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias -where 1 = 1 - -), __dbt__cte__unnest_alias_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__unnest_alias_ab1 --- unnest_alias -where 1 = 1 - -), __dbt__cte__unnest_alias_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(children as varchar), '') as varchar)) as _airbyte_unnest_alias_hashid, - tmp.* -from __dbt__cte__unnest_alias_ab2 tmp --- unnest_alias -where 1 = 1 - -)-- Final base SQL model -select - id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from __dbt__cte__unnest_alias_ab3 --- unnest_alias from "integrationtests".test_normalization._airbyte_raw_unnest_alias -where 1 = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 6a49f0ff964e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,54 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_array__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__conflict_stream_array_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - json_extract_path_text(_airbyte_data, 'conflict_stream_array', true) as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -where 1 = 1 -), __dbt__cte__conflict_stream_array_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__conflict_stream_array_ab1 --- conflict_stream_array -where 1 = 1 -), __dbt__cte__conflict_stream_array_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_array as varchar), '') as varchar)) as _airbyte_conflict_stream_array_hashid, - tmp.* -from __dbt__cte__conflict_stream_array_ab2 tmp --- conflict_stream_array -where 1 = 1 -)-- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from __dbt__cte__conflict_stream_array_ab3 --- conflict_stream_array from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index 13aba9129a52..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,56 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_name__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__conflict_stream_name_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - - case when json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) end - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -where 1 = 1 -), __dbt__cte__conflict_stream_name_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(conflict_stream_name as varchar) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__conflict_stream_name_ab1 --- conflict_stream_name -where 1 = 1 -), __dbt__cte__conflict_stream_name_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_hashid, - tmp.* -from __dbt__cte__conflict_stream_name_ab2 tmp --- conflict_stream_name -where 1 = 1 -)-- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from __dbt__cte__conflict_stream_name_ab3 --- conflict_stream_name from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index bab7f7d138d5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,57 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - case when json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) end - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null -), __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as varchar) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -), __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -)-- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 --- conflict_stream_name at conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 19ac73995aa9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name_conflict_stream_name__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - case when json_extract_path_text(conflict_stream_name, 'groups', true) != '' then json_extract_path_text(conflict_stream_name, 'groups', true) end as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null -), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as varchar) as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as varchar), '') || '-' || coalesce(cast(groups as varchar), '') as varchar)) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -)-- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - groups, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index bd6a7edaa276..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,56 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_scalar__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__conflict_stream_scalar_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) != '' then json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) end as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -where 1 = 1 -), __dbt__cte__conflict_stream_scalar_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(conflict_stream_scalar as - bigint -) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__conflict_stream_scalar_ab1 --- conflict_stream_scalar -where 1 = 1 -), __dbt__cte__conflict_stream_scalar_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_scalar as varchar), '') as varchar)) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from __dbt__cte__conflict_stream_scalar_ab2 tmp --- conflict_stream_scalar -where 1 = 1 -)-- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from __dbt__cte__conflict_stream_scalar_ab3 --- conflict_stream_scalar from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index f691cc5da8fb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,61 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - json_extract_path_text("partition", 'double_array_data', true) as double_array_data, - json_extract_path_text("partition", 'DATA', true) as data, - json_extract_path_text("partition", 'column`_''with"_quotes', true) as "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and "partition" is not null -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - data, - "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as varchar), '') || '-' || coalesce(cast(double_array_data as varchar), '') || '-' || coalesce(cast(data as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') as varchar)) as _airbyte_partition_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -)-- Final base SQL model -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - data, - "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_partition_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql deleted file mode 100644 index 9eb1e109077e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ /dev/null @@ -1,100 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - -), -joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - json_extract_array_element_text("column`_'with""_quotes", numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition."column`_'with""_quotes" - where numbers.generated_number <= json_array_length("column`_'with""_quotes", true) -) -select - _airbyte_partition_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and "column`_'with""_quotes" is not null -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as varchar) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_column___with__quotes_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 -)-- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql deleted file mode 100644 index ed667f0e4087..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ /dev/null @@ -1,100 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_data__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - -), -joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - json_extract_array_element_text(data, numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.data - where numbers.generated_number <= json_array_length(data, true) -) -select - _airbyte_partition_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and data is not null -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as varchar) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_data_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 tmp --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 -)-- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_data_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index 51f7dcf22ca2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,100 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - -), -joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - json_extract_array_element_text(double_array_data, numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.double_array_data - where numbers.generated_number <= json_array_length(double_array_data, true) -) -select - _airbyte_partition_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'id', true) end as id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and double_array_data is not null -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as varchar) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(id as varchar), '') as varchar)) as _airbyte_double_array_data_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 -)-- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql deleted file mode 100644 index 95f350d3abdd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,54 +0,0 @@ - - - create table - "integrationtests".test_normalization."non_nested_stream_without_namespace_resulting_into_long_names__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 -), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(date as varchar) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 -), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, - tmp.* -from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 tmp --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 -)-- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid -from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 --- non_nested_stream_without_namespace_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index aca1269c8d5f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,107 +0,0 @@ - - - create table - "integrationtests".test_normalization."unnest_alias_children__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__unnest_alias_children_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - -), -joined as ( - select - _airbyte_unnest_alias_hashid as _airbyte_hashid, - json_extract_array_element_text(children, numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."unnest_alias" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in unnest_alias.children - where numbers.generated_number <= json_array_length(children, true) -) -select - _airbyte_unnest_alias_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'ab_id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'ab_id', true) end as ab_id, - - case when json_extract_path_text(_airbyte_nested_data, 'owner', true) != '' then json_extract_path_text(_airbyte_nested_data, 'owner', true) end - as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."unnest_alias" as table_alias --- children at unnest_alias/children -left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid -where 1 = 1 -and children is not null -), __dbt__cte__unnest_alias_children_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - bigint -) as ab_id, - cast(owner as varchar) as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__unnest_alias_children_ab1 --- children at unnest_alias/children -where 1 = 1 -), __dbt__cte__unnest_alias_children_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as varchar), '') || '-' || coalesce(cast(ab_id as varchar), '') || '-' || coalesce(cast(owner as varchar), '') as varchar)) as _airbyte_children_hashid, - tmp.* -from __dbt__cte__unnest_alias_children_ab2 tmp --- children at unnest_alias/children -where 1 = 1 -)-- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_children_hashid -from __dbt__cte__unnest_alias_children_ab3 --- children at unnest_alias/children from "integrationtests".test_normalization."unnest_alias" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index d6c293674dbe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,57 +0,0 @@ - - - create table - "integrationtests".test_normalization."unnest_alias_children_owner__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__unnest_alias_children_owner_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - case when json_extract_path_text(owner, 'owner_id', true) != '' then json_extract_path_text(owner, 'owner_id', true) end as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."unnest_alias_children" as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and owner is not null -), __dbt__cte__unnest_alias_children_owner_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - bigint -) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__unnest_alias_children_owner_ab1 --- owner at unnest_alias/children/owner -where 1 = 1 -), __dbt__cte__unnest_alias_children_owner_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_children_hashid as varchar), '') || '-' || coalesce(cast(owner_id as varchar), '') as varchar)) as _airbyte_owner_hashid, - tmp.* -from __dbt__cte__unnest_alias_children_owner_ab2 tmp --- owner at unnest_alias/children/owner -where 1 = 1 -)-- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_owner_hashid -from __dbt__cte__unnest_alias_children_owner_ab3 --- owner at unnest_alias/children/owner from "integrationtests".test_normalization."unnest_alias_children" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql deleted file mode 100644 index 69dff6931661..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,54 +0,0 @@ - - - create table - "integrationtests".test_normalization_namespace."simple_stream_with_namespace_resulting_into_long_names__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 -), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(date as varchar) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 -), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, - tmp.* -from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 tmp --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 -)-- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid -from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 --- simple_stream_with_namespace_resulting_into_long_names from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql deleted file mode 100644 index 8338f7a9440b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql deleted file mode 100644 index 47b4d187fc92..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_array_ab1') }} --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql deleted file mode 100644 index d4efd628548e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - array_to_string('conflict_stream_array'), - ]) }} as _airbyte_conflict_stream_array_hashid, - tmp.* -from {{ ref('conflict_stream_array_ab2') }} tmp --- conflict_stream_array -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql deleted file mode 100644 index 223381444e1f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql deleted file mode 100644 index 02a958942c9c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_ab1') }} --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql deleted file mode 100644 index 89a2bdb2f1ed..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'conflict_stream_name', - ]) }} as _airbyte_conflict_stream_name_hashid, - tmp.* -from {{ ref('conflict_stream_name_ab2') }} tmp --- conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index a8a342548c36..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name') }} as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index 3c458a14224f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index aa0d9c1cb20b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_conflict_stream_name_hashid', - 'conflict_stream_name', - ]) }} as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql deleted file mode 100644 index d69a8acbadec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql deleted file mode 100644 index dc826e7f53c2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as {{ dbt_utils.type_string() }}) as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql deleted file mode 100644 index 9b7dfbfccab7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_conflict_stream_name_2_hashid', - 'groups', - ]) }} as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2') }} tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql deleted file mode 100644 index 65a84615e748..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql deleted file mode 100644 index 6205a85370ec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('conflict_stream_scalar_ab1') }} --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql deleted file mode 100644 index 42f6c144d3c6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'conflict_stream_scalar', - ]) }} as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from {{ ref('conflict_stream_scalar_ab2') }} tmp --- conflict_stream_scalar -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql index 58d09347649a..a4a22de48c30 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql index 9369b7333ece..ecccba133d50 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql index fc17127ac094..1e4d83af5f23 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -9,12 +9,12 @@ select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as data, - {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} as table_alias -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 and {{ adapter.quote('partition') }} is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql deleted file mode 100644 index aa1074726da6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - data, - {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1') }} --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql deleted file mode 100644 index 43e9f45bf5f1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid', - array_to_string('double_array_data'), - array_to_string('data'), - array_to_string(adapter.quote('column`_\'with""_quotes')), - ]) }} as _airbyte_partition_hashid, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2') }} tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql deleted file mode 100644 index f3478375aeaf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', adapter.quote('column`_\'with""_quotes')) }} -select - _airbyte_partition_hashid, - {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} -where 1 = 1 -and {{ adapter.quote('column`_\'with""_quotes') }} is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql deleted file mode 100644 index ae54378f3109..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1') }} --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql deleted file mode 100644 index 0fe19cecbd7c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'currency', - ]) }} as _airbyte_column___with__quotes_hashid, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2') }} tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql index 791d76b4f4e3..eb49c6cbff75 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql @@ -1,6 +1,5 @@ {{ config( sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -17,4 +16,5 @@ from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partit {{ cross_join_unnest('partition', 'data') }} where 1 = 1 and data is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql deleted file mode 100644 index 01ca205eece7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1') }} --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql deleted file mode 100644 index 377ca6ed0320..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'currency', - ]) }} as _airbyte_data_hashid, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2') }} tmp --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql index 5268dd0dbd4f..4e16e9d3b93c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -1,6 +1,5 @@ {{ config( sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "nested-intermediate" ] ) }} @@ -17,4 +16,5 @@ from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partit {{ cross_join_unnest('partition', 'double_array_data') }} where 1 = 1 and double_array_data is not null +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql deleted file mode 100644 index cb8eb9675598..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1') }} --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql deleted file mode 100644 index 3972432d4691..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_partition_hashid', - 'id', - ]) }} as _airbyte_double_array_data_hashid, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2') }} tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql deleted file mode 100644 index a80e0a1c756b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql deleted file mode 100644 index a8aee99469e5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab1') }} --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql deleted file mode 100644 index f55f72315f8a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'date', - ]) }} as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, - tmp.* -from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab2') }} tmp --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql deleted file mode 100644 index d4f394ed7911..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql deleted file mode 100644 index 2ccb9349d718..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_ab1') }} --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql deleted file mode 100644 index 68c9c6064d65..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - array_to_string('children'), - ]) }} as _airbyte_unnest_alias_hashid, - tmp.* -from {{ ref('unnest_alias_ab2') }} tmp --- unnest_alias -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql deleted file mode 100644 index 1a67101d4ea8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} -select - _airbyte_unnest_alias_hashid, - {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, - {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias') }} as table_alias --- children at unnest_alias/children -{{ cross_join_unnest('unnest_alias', 'children') }} -where 1 = 1 -and children is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql deleted file mode 100644 index 4c4c2367a5f6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, - cast(owner as {{ type_json() }}) as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children_ab1') }} --- children at unnest_alias/children -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql deleted file mode 100644 index 06e2a4e3a52e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_unnest_alias_hashid', - 'ab_id', - 'owner', - ]) }} as _airbyte_children_hashid, - tmp.* -from {{ ref('unnest_alias_children_ab2') }} tmp --- children at unnest_alias/children -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql deleted file mode 100644 index 0ed42cb28690..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - {{ json_extract_scalar('owner', ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children') }} as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and owner is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql deleted file mode 100644 index f6b016bf5e78..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('unnest_alias_children_owner_ab1') }} --- owner at unnest_alias/children/owner -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql deleted file mode 100644 index 1a77db63b778..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_airbyte_children_hashid', - 'owner_id', - ]) }} as _airbyte_owner_hashid, - tmp.* -from {{ ref('unnest_alias_children_owner_ab2') }} tmp --- owner at unnest_alias/children/owner -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql deleted file mode 100644 index 7ae7824eab31..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql deleted file mode 100644 index 3f8824570d2d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab1') }} --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql deleted file mode 100644 index 0b1c938f2e3e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization_namespace", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'date', - ]) }} as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, - tmp.* -from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab2') }} tmp --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 80606e29b92c..07acb1644158 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -1,5 +1,5 @@ {{ config( - sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + sort = ["_airbyte_active_row", "_airbyte_unique_key_scd", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", tags = [ "top-level" ] @@ -63,13 +63,13 @@ scd_data as ( date desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(date) over ( + case when row_number() over ( partition by id order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..a2c049e30458 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,20 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + data, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_partition_hashid +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3') }} +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql new file mode 100644 index 000000000000..c276d29dfb77 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -0,0 +1,18 @@ +{{ config( + sort = "_airbyte_emitted_at", + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_data_hashid +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3') }} +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..3b9f5b35989b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,18 @@ +{{ config( + sort = "_airbyte_emitted_at", + schema = "test_normalization", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3') }} +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index 10f4b807149e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - children, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 328153a742fe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from {{ ref('conflict_stream_array_ab3') }} --- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index 51af86be3918..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from {{ ref('conflict_stream_name_ab3') }} --- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index df29869e1b30..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 78325474c27a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - groups, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3') }} --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index e00b45e8686a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from {{ ref('conflict_stream_scalar_ab3') }} --- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index 6e3926b47284..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - data, - {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_partition_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3') }} --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql deleted file mode 100644 index e8d0f08669f7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3') }} --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql deleted file mode 100644 index 2186017e2c85..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_data_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3') }} --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index 7a2ff26eb0f0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3') }} --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql deleted file mode 100644 index c106d56f953c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid -from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab3') }} --- non_nested_stream_without_namespace_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index 9c5ef61b6354..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_children_hashid -from {{ ref('unnest_alias_children_ab3') }} --- children at unnest_alias/children from {{ ref('unnest_alias') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index 322eb9d5f0f6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_owner_hashid -from {{ ref('unnest_alias_children_owner_ab3') }} --- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql deleted file mode 100644 index 08f4dde7545a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization_namespace", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid -from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab3') }} --- simple_stream_with_namespace_resulting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql index aaf10fa8e809..6a471583d5ec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..4cb82875daea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" ("_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid") + ( + select "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid" + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql new file mode 100644 index 000000000000..6ce45aa678ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -0,0 +1,9 @@ + + + + insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" ("_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid") + ( + select "_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid" + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..491418bbabac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,9 @@ + + + + insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" ("_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid") + ( + select "_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid" + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql deleted file mode 100644 index 3c3524996abe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "integrationtests".test_normalization."unnest_alias" - where (_airbyte_ab_id) in ( - select (_airbyte_ab_id) - from "unnest_alias__dbt_tmp" - ); - - insert into "integrationtests".test_normalization."unnest_alias" ("id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid") - ( - select "id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid" - from "unnest_alias__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql deleted file mode 100644 index 6a49f0ff964e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql +++ /dev/null @@ -1,54 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_array__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__conflict_stream_array_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - json_extract_path_text(_airbyte_data, 'conflict_stream_array', true) as conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array -where 1 = 1 -), __dbt__cte__conflict_stream_array_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__conflict_stream_array_ab1 --- conflict_stream_array -where 1 = 1 -), __dbt__cte__conflict_stream_array_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_array as varchar), '') as varchar)) as _airbyte_conflict_stream_array_hashid, - tmp.* -from __dbt__cte__conflict_stream_array_ab2 tmp --- conflict_stream_array -where 1 = 1 -)-- Final base SQL model -select - id, - conflict_stream_array, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_conflict_stream_array_hashid -from __dbt__cte__conflict_stream_array_ab3 --- conflict_stream_array from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql deleted file mode 100644 index 13aba9129a52..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql +++ /dev/null @@ -1,56 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_name__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__conflict_stream_name_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - - case when json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) end - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name -where 1 = 1 -), __dbt__cte__conflict_stream_name_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(conflict_stream_name as varchar) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__conflict_stream_name_ab1 --- conflict_stream_name -where 1 = 1 -), __dbt__cte__conflict_stream_name_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_hashid, - tmp.* -from __dbt__cte__conflict_stream_name_ab2 tmp --- conflict_stream_name -where 1 = 1 -)-- Final base SQL model -select - id, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_hashid -from __dbt__cte__conflict_stream_name_ab3 --- conflict_stream_name from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index bab7f7d138d5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,57 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_hashid, - - case when json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) end - as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null -), __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_hashid, - cast(conflict_stream_name as varchar) as conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -), __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_2_hashid, - tmp.* -from __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name -where 1 = 1 -)-- Final base SQL model -select - _airbyte_conflict_stream_name_hashid, - conflict_stream_name, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_2_hashid -from __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 --- conflict_stream_name at conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql deleted file mode 100644 index 19ac73995aa9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ /dev/null @@ -1,55 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name_conflict_stream_name__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_conflict_stream_name_2_hashid, - case when json_extract_path_text(conflict_stream_name, 'groups', true) != '' then json_extract_path_text(conflict_stream_name, 'groups', true) end as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and conflict_stream_name is not null -), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_conflict_stream_name_2_hashid, - cast(groups as varchar) as groups, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as varchar), '') || '-' || coalesce(cast(groups as varchar), '') as varchar)) as _airbyte_conflict_stream_name_3_hashid, - tmp.* -from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -)-- Final base SQL model -select - _airbyte_conflict_stream_name_2_hashid, - groups, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_conflict_stream_name_3_hashid -from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql deleted file mode 100644 index bd6a7edaa276..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ /dev/null @@ -1,56 +0,0 @@ - - - create table - "integrationtests".test_normalization."conflict_stream_scalar__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__conflict_stream_scalar_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) != '' then json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) end as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar -where 1 = 1 -), __dbt__cte__conflict_stream_scalar_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(conflict_stream_scalar as - bigint -) as conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__conflict_stream_scalar_ab1 --- conflict_stream_scalar -where 1 = 1 -), __dbt__cte__conflict_stream_scalar_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_scalar as varchar), '') as varchar)) as _airbyte_conflict_stream_scalar_hashid, - tmp.* -from __dbt__cte__conflict_stream_scalar_ab2 tmp --- conflict_stream_scalar -where 1 = 1 -)-- Final base SQL model -select - id, - conflict_stream_scalar, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_conflict_stream_scalar_hashid -from __dbt__cte__conflict_stream_scalar_ab3 --- conflict_stream_scalar from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index f691cc5da8fb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,61 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - json_extract_path_text("partition", 'double_array_data', true) as double_array_data, - json_extract_path_text("partition", 'DATA', true) as data, - json_extract_path_text("partition", 'column`_''with"_quotes', true) as "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and "partition" is not null -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - data, - "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as varchar), '') || '-' || coalesce(cast(double_array_data as varchar), '') || '-' || coalesce(cast(data as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') as varchar)) as _airbyte_partition_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -)-- Final base SQL model -select - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - double_array_data, - data, - "column`_'with""_quotes", - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_partition_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql deleted file mode 100644 index 9eb1e109077e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ /dev/null @@ -1,100 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - -), -joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - json_extract_array_element_text("column`_'with""_quotes", numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition."column`_'with""_quotes" - where numbers.generated_number <= json_array_length("column`_'with""_quotes", true) -) -select - _airbyte_partition_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and "column`_'with""_quotes" is not null -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as varchar) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_column___with__quotes_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 -)-- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_column___with__quotes_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql deleted file mode 100644 index ed667f0e4087..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ /dev/null @@ -1,100 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_data__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 1 - order by generated_number - - -), -joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - json_extract_array_element_text(data, numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.data - where numbers.generated_number <= json_array_length(data, true) -) -select - _airbyte_partition_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and data is not null -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(currency as varchar) as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_data_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 tmp --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 -)-- Final base SQL model -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_data_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index 51f7dcf22ca2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,100 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - -), -joined as ( - select - _airbyte_partition_hashid as _airbyte_hashid, - json_extract_array_element_text(double_array_data, numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.double_array_data - where numbers.generated_number <= json_array_length(double_array_data, true) -) -select - _airbyte_partition_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'id', true) end as id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where 1 = 1 -and double_array_data is not null -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_partition_hashid, - cast(id as varchar) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(id as varchar), '') as varchar)) as _airbyte_double_array_data_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 -)-- Final base SQL model -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql deleted file mode 100644 index 95f350d3abdd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,54 +0,0 @@ - - - create table - "integrationtests".test_normalization."non_nested_stream_without_namespace_resulting_into_long_names__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 -), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(date as varchar) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 -), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, - tmp.* -from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 tmp --- non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 -)-- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid -from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 --- non_nested_stream_without_namespace_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql deleted file mode 100644 index aca1269c8d5f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql +++ /dev/null @@ -1,107 +0,0 @@ - - - create table - "integrationtests".test_normalization."unnest_alias_children__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__unnest_alias_children_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -with numbers as ( - - - - - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( - - select - - - p0.generated_number * power(2, 0) - - - + 1 - as generated_number - - from - - - p as p0 - - - - ) - - select * - from unioned - where generated_number <= 2 - order by generated_number - - -), -joined as ( - select - _airbyte_unnest_alias_hashid as _airbyte_hashid, - json_extract_array_element_text(children, numbers.generated_number::int - 1, true) as _airbyte_nested_data - from "integrationtests".test_normalization."unnest_alias" - cross join numbers - -- only generate the number of records in the cross join that corresponds - -- to the number of items in unnest_alias.children - where numbers.generated_number <= json_array_length(children, true) -) -select - _airbyte_unnest_alias_hashid, - case when json_extract_path_text(_airbyte_nested_data, 'ab_id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'ab_id', true) end as ab_id, - - case when json_extract_path_text(_airbyte_nested_data, 'owner', true) != '' then json_extract_path_text(_airbyte_nested_data, 'owner', true) end - as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."unnest_alias" as table_alias --- children at unnest_alias/children -left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid -where 1 = 1 -and children is not null -), __dbt__cte__unnest_alias_children_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_unnest_alias_hashid, - cast(ab_id as - bigint -) as ab_id, - cast(owner as varchar) as owner, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__unnest_alias_children_ab1 --- children at unnest_alias/children -where 1 = 1 -), __dbt__cte__unnest_alias_children_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as varchar), '') || '-' || coalesce(cast(ab_id as varchar), '') || '-' || coalesce(cast(owner as varchar), '') as varchar)) as _airbyte_children_hashid, - tmp.* -from __dbt__cte__unnest_alias_children_ab2 tmp --- children at unnest_alias/children -where 1 = 1 -)-- Final base SQL model -select - _airbyte_unnest_alias_hashid, - ab_id, - owner, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_children_hashid -from __dbt__cte__unnest_alias_children_ab3 --- children at unnest_alias/children from "integrationtests".test_normalization."unnest_alias" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql deleted file mode 100644 index d6c293674dbe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ /dev/null @@ -1,57 +0,0 @@ - - - create table - "integrationtests".test_normalization."unnest_alias_children_owner__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__unnest_alias_children_owner_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _airbyte_children_hashid, - case when json_extract_path_text(owner, 'owner_id', true) != '' then json_extract_path_text(owner, 'owner_id', true) end as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."unnest_alias_children" as table_alias --- owner at unnest_alias/children/owner -where 1 = 1 -and owner is not null -), __dbt__cte__unnest_alias_children_owner_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _airbyte_children_hashid, - cast(owner_id as - bigint -) as owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__unnest_alias_children_owner_ab1 --- owner at unnest_alias/children/owner -where 1 = 1 -), __dbt__cte__unnest_alias_children_owner_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_airbyte_children_hashid as varchar), '') || '-' || coalesce(cast(owner_id as varchar), '') as varchar)) as _airbyte_owner_hashid, - tmp.* -from __dbt__cte__unnest_alias_children_owner_ab2 tmp --- owner at unnest_alias/children/owner -where 1 = 1 -)-- Final base SQL model -select - _airbyte_children_hashid, - owner_id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_owner_hashid -from __dbt__cte__unnest_alias_children_owner_ab3 --- owner at unnest_alias/children/owner from "integrationtests".test_normalization."unnest_alias_children" -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql deleted file mode 100644 index 69dff6931661..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ /dev/null @@ -1,54 +0,0 @@ - - - create table - "integrationtests".test_normalization_namespace."simple_stream_with_namespace_resulting_into_long_names__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 -), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(date as varchar) as date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 -), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, - tmp.* -from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 tmp --- simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 -)-- Final base SQL model -select - id, - date, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid -from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 --- simple_stream_with_namespace_resulting_into_long_names from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml new file mode 100755 index 000000000000..522b1e595e8a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["modified_models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml new file mode 100644 index 000000000000..9ad815875900 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 354cbd13be02..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,77 +0,0 @@ - - - - create table - "integrationtests"."test_normalization"."dedup_cdc_excluded_scd__dbt_tmp" - - - compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) - as ( - -with - -input_data as ( - select * - from "integrationtests"._airbyte_test_normalization."dedup_cdc_excluded_ab3" - -- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar), cast(_ab_cdc_updated_at as varchar) - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index c9ec8d0622ff..621559264ec8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -5,7 +5,7 @@ "integrationtests"."test_normalization"."dedup_exchange_rate_scd__dbt_tmp" - compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + compound sortkey(_airbyte_active_row,_airbyte_unique_key_scd,_airbyte_emitted_at) as ( with @@ -36,13 +36,13 @@ scd_data as ( date desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(date) over ( + case when row_number() over ( partition by id, currency, cast(nzd as varchar) order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index c63c52ddd0e7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,69 +0,0 @@ - - - - create table - "integrationtests"."test_normalization"."renamed_dedup_cdc_excluded_scd__dbt_tmp" - - - compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) - as ( - -with - -input_data as ( - select * - from "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" - -- renamed_dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_renamed_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 688fbdcbe378..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,29 +0,0 @@ - - - - create table - "integrationtests"."test_normalization"."dedup_cdc_excluded__dbt_tmp" - - - compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from "integrationtests".test_normalization."dedup_cdc_excluded_scd" --- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded -where 1 = 1 -and _airbyte_active_row = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index a560a42d546a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,90 +0,0 @@ - - - - create table - "integrationtests"."test_normalization"."exchange_rate__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - as ( - -with __dbt__cte__exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, - case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", - case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, - case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, - case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 - -), __dbt__cte__exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(currency as varchar) as currency, - cast(nullif(date, '') as - date -) as date, - cast(nullif(timestamp_col, '') as - timestamp with time zone -) as timestamp_col, - cast("hkd@spéçiäl & characters" as - float -) as "hkd@spéçiäl & characters", - cast(hkd_special___characters as varchar) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__exchange_rate_ab1 --- exchange_rate -where 1 = 1 - -), __dbt__cte__exchange_rate_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(hkd_special___characters as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') as varchar)) as _airbyte_exchange_rate_hashid, - tmp.* -from __dbt__cte__exchange_rate_ab2 tmp --- exchange_rate -where 1 = 1 - -)-- Final base SQL model -select - id, - currency, - date, - timestamp_col, - "hkd@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "integrationtests".test_normalization._airbyte_raw_exchange_rate -where 1 = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 035acaa17f2a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - - create table - "integrationtests"."test_normalization"."renamed_dedup_cdc_excluded__dbt_tmp" - - - compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" --- renamed_dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded -where 1 = 1 -and _airbyte_active_row = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index a9657af6c9d3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,77 +0,0 @@ - - - create table - "integrationtests".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - - - compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) - as ( - -with - -input_data as ( - select * - from "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3" - -- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar), cast(_ab_cdc_updated_at as varchar), cast(_ab_cdc_log_pos as varchar) - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..6ac73c8b3ed3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,87 @@ + + + create table + "integrationtests".test_normalization."exchange_rate__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, + case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", + case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, + case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, + case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, + case when json_extract_path_text(_airbyte_data, 'column`_''with"_quotes', true) != '' then json_extract_path_text(_airbyte_data, 'column`_''with"_quotes', true) end as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as varchar) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("hkd@spéçiäl & characters" as + float +) as "hkd@spéçiäl & characters", + cast(hkd_special___characters as varchar) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast("column`_'with""_quotes" as varchar) as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(hkd_special___characters as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') as varchar)) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__cte__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + "hkd@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__cte__exchange_rate_ab3 +-- exchange_rate from "integrationtests".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 6bc840ab2ff4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - create table - "integrationtests".test_normalization."pos_dedup_cdcx__dbt_tmp" - - - compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from "integrationtests".test_normalization."pos_dedup_cdcx_scd" --- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index aef3d5e083d6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,53 +0,0 @@ - - - create view "integrationtests"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( - -with __dbt__cte__dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 - -), __dbt__cte__dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as varchar) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__dedup_cdc_excluded_ab1 --- dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__dedup_cdc_excluded_ab2 tmp --- dedup_cdc_excluded -where 1 = 1 - - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 3502913a583f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,54 +0,0 @@ - - - create view "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( - -with __dbt__cte__pos_dedup_cdcx_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) end as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1 -), __dbt__cte__pos_dedup_cdcx_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as varchar) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__pos_dedup_cdcx_ab1 --- pos_dedup_cdcx -where 1 = 1 -)-- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from __dbt__cte__pos_dedup_cdcx_ab2 tmp --- pos_dedup_cdcx -where 1 = 1 - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index e7841d06e0fe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,39 +0,0 @@ - - - create view "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( - -with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - -), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__renamed_dedup_cdc_excluded_ab1 --- renamed_dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql deleted file mode 100644 index 98cd148ff512..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql deleted file mode 100644 index b808389ede5a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(name as {{ dbt_utils.type_string() }}) as name, - cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('dedup_cdc_excluded_ab1') }} --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 463aebe85f6b..4ed56a4f4685 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 35914dac7dcc..b3d32dd357c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql deleted file mode 100644 index 354a14427e50..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, - {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as {{ adapter.quote('hkd@spéçiäl & characters') }}, - {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, - {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, - {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql deleted file mode 100644 index 8096d935aeaa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - cast({{ empty_string_to_null('date') }} as {{ type_date() }}) as date, - cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col, - cast({{ adapter.quote('hkd@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('hkd@spéçiäl & characters') }}, - cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, - cast(nzd as {{ dbt_utils.type_float() }}) as nzd, - cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('exchange_rate_ab1') }} --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql deleted file mode 100644 index a07ee57a28a3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'date', - 'timestamp_col', - adapter.quote('hkd@spéçiäl & characters'), - 'hkd_special___characters', - 'nzd', - 'usd', - ]) }} as _airbyte_exchange_rate_hashid, - tmp.* -from {{ ref('exchange_rate_ab2') }} tmp --- exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql deleted file mode 100644 index 9d9592bf5bcc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql deleted file mode 100644 index 32dea2cfa7c8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(name as {{ dbt_utils.type_string() }}) as name, - cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('pos_dedup_cdcx_ab1') }} --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql deleted file mode 100644 index c324b5203905..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql deleted file mode 100644 index df5c1035dd92..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('renamed_dedup_cdc_excluded_ab1') }} --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 19d4d181c449..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,112 +0,0 @@ -{{ config( - sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('dedup_cdc_excluded_ab3') }} - -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('dedup_cdc_excluded_ab3') }} - -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 12f992cd0bbd..d822427c21c6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,5 +1,5 @@ {{ config( - sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + sort = ["_airbyte_active_row", "_airbyte_unique_key_scd", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", tags = [ "top-level" ] @@ -72,13 +72,13 @@ scd_data as ( date desc, _airbyte_emitted_at desc ) as _airbyte_end_at, - case when lag(date) over ( + case when row_number() over ( partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, + ) = 1 then 1 else 0 end as _airbyte_active_row, _airbyte_ab_id, _airbyte_emitted_at, _airbyte_dedup_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 500c257b4a70..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,104 +0,0 @@ -{{ config( - sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} - -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('renamed_dedup_cdc_excluded_ab3') }} - -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_renamed_dedup_cdc_excluded_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 385a2a1c6b24..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where 1 = 1 -and _airbyte_active_row = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 62dbd07d0962..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 12a26f6e4326..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_renamed_dedup_cdc_excluded_hashid -from {{ ref('renamed_dedup_cdc_excluded_scd') }} --- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} -where 1 = 1 -and _airbyte_active_row = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 5140caa6d9f4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,114 +0,0 @@ -{{ config( - sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at') }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data - union all - select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('pos_dedup_cdcx_ab3') }} - -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) - order by _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..abe73769b044 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,25 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 1786654bece8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_scd') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where 1 = 1 -and _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 1e63c2ad6c92..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index e92210d17f06..10aa8025dfce 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + unique_key = '_airbyte_ab_id', schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index e82906073868..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - '_ab_cdc_log_pos', - ]) }} as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from {{ ref('pos_dedup_cdcx_ab2') }} tmp --- pos_dedup_cdcx -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index b59cdd1d2b2c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp --- renamed_dedup_cdc_excluded -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..a10b7ccffd0b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,24 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, + {{ json_extract_scalar('_airbyte_data', ['new_column'], ['new_column']) }} as new_column, + {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, + {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, + {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as {{ adapter.quote('hkd@spéçiäl & characters') }}, + {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, + {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..a16472d56938 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,24 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_float() }}) as id, + cast(currency as {{ dbt_utils.type_string() }}) as currency, + cast(new_column as {{ dbt_utils.type_float() }}) as new_column, + cast({{ empty_string_to_null('date') }} as {{ type_date() }}) as date, + cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col, + cast({{ adapter.quote('hkd@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('hkd@spéçiäl & characters') }}, + cast(nzd as {{ dbt_utils.type_float() }}) as nzd, + cast(usd as {{ dbt_utils.type_bigint() }}) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('dedup_exchange_rate_ab1') }} +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..f611e335bc09 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,122 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key_scd", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key, + id, + currency, + new_column, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + nzd, + usd, + date as _airbyte_start_at, + lag(date) over ( + partition by cast(id as {{ dbt_utils.type_string() }}), currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when row_number() over ( + partition by cast(id as {{ dbt_utils.type_string() }}), currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) = 1 then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + new_column, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..8a0daa8d52db --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,27 @@ +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + new_column, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from {{ ref('dedup_exchange_rate_scd') }} +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..5b4d6b1970b7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,25 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = '_airbyte_ab_id', + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + new_column, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + nzd, + usd, + {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..fdd6d936eeed --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,24 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'new_column', + 'date', + 'timestamp_col', + adapter.quote('hkd@spéçiäl & characters'), + 'nzd', + 'usd', + ]) }} as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml new file mode 100644 index 000000000000..dd538a80131a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml @@ -0,0 +1,11 @@ +version: 2 +sources: +- name: test_normalization + quoting: + database: true + schema: false + identifier: false + tables: + - name: _airbyte_raw_dedup_exchange_rate + - name: _airbyte_raw_exchange_rate + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index b9485899f9c3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "integrationtests".test_normalization."dedup_cdc_excluded_scd" - where (_airbyte_unique_key_scd) in ( - select (_airbyte_unique_key_scd) - from "dedup_cdc_excluded_scd__dbt_tmp" - ); - - insert into "integrationtests".test_normalization."dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") - ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" - from "dedup_cdc_excluded_scd__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 70ebdfca1c57..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" - where (_airbyte_unique_key_scd) in ( - select (_airbyte_unique_key_scd) - from "renamed_dedup_cdc_excluded_scd__dbt_tmp" - ); - - insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") - ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" - from "renamed_dedup_cdc_excluded_scd__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 8f1ca62ed990..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "integrationtests".test_normalization."dedup_cdc_excluded" - where (_airbyte_unique_key) in ( - select (_airbyte_unique_key) - from "dedup_cdc_excluded__dbt_tmp" - ); - - insert into "integrationtests".test_normalization."dedup_cdc_excluded" ("_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") - ( - select "_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" - from "dedup_cdc_excluded__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 0da80f1f8245..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "integrationtests".test_normalization."exchange_rate" - where (_airbyte_ab_id) in ( - select (_airbyte_ab_id) - from "exchange_rate__dbt_tmp" - ); - - insert into "integrationtests".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") - ( - select "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid" - from "exchange_rate__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 94ffdc307498..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "integrationtests".test_normalization."renamed_dedup_cdc_excluded" - where (_airbyte_unique_key) in ( - select (_airbyte_unique_key) - from "renamed_dedup_cdc_excluded__dbt_tmp" - ); - - insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") - ( - select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" - from "renamed_dedup_cdc_excluded__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index a9657af6c9d3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,77 +0,0 @@ - - - create table - "integrationtests".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - - - compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) - as ( - -with - -input_data as ( - select * - from "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3" - -- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by - _airbyte_emitted_at is null asc, - _airbyte_emitted_at desc, - _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar), cast(_ab_cdc_updated_at as varchar), cast(_ab_cdc_log_pos as varchar) - order by _airbyte_ab_id - ) as _airbyte_row_num, - md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from dedup_data where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..6ac73c8b3ed3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,87 @@ + + + create table + "integrationtests".test_normalization."exchange_rate__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, + case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", + case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, + case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, + case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, + case when json_extract_path_text(_airbyte_data, 'column`_''with"_quotes', true) != '' then json_extract_path_text(_airbyte_data, 'column`_''with"_quotes', true) end as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as varchar) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("hkd@spéçiäl & characters" as + float +) as "hkd@spéçiäl & characters", + cast(hkd_special___characters as varchar) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast("column`_'with""_quotes" as varchar) as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(hkd_special___characters as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') as varchar)) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__cte__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + "hkd@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__cte__exchange_rate_ab3 +-- exchange_rate from "integrationtests".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql deleted file mode 100644 index 6bc840ab2ff4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ /dev/null @@ -1,27 +0,0 @@ - - - create table - "integrationtests".test_normalization."pos_dedup_cdcx__dbt_tmp" - - - compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) - as ( - --- Final base SQL model -select - _airbyte_unique_key, - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_pos_dedup_cdcx_hashid -from "integrationtests".test_normalization."pos_dedup_cdcx_scd" --- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx -where 1 = 1 -and _airbyte_active_row = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index aef3d5e083d6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,53 +0,0 @@ - - - create view "integrationtests"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( - -with __dbt__cte__dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -where 1 = 1 - -), __dbt__cte__dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as varchar) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__dedup_cdc_excluded_ab1 --- dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__dedup_cdc_excluded_ab2 tmp --- dedup_cdc_excluded -where 1 = 1 - - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql deleted file mode 100644 index 3502913a583f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql +++ /dev/null @@ -1,54 +0,0 @@ - - - create view "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( - -with __dbt__cte__pos_dedup_cdcx_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) end as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx -where 1 = 1 -), __dbt__cte__pos_dedup_cdcx_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as varchar) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - cast(_ab_cdc_log_pos as - float -) as _ab_cdc_log_pos, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__pos_dedup_cdcx_ab1 --- pos_dedup_cdcx -where 1 = 1 -)-- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_pos_dedup_cdcx_hashid, - tmp.* -from __dbt__cte__pos_dedup_cdcx_ab2 tmp --- pos_dedup_cdcx -where 1 = 1 - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index e7841d06e0fe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,39 +0,0 @@ - - - create view "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( - -with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - -), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__renamed_dedup_cdc_excluded_ab1 --- renamed_dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql deleted file mode 100644 index 3fb70e64ecc7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" - where (_airbyte_unique_key_scd) in ( - select (_airbyte_unique_key_scd) - from "renamed_dedup_cdc_excluded_scd__dbt_tmp" - ); - - insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") - ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" - from "renamed_dedup_cdc_excluded_scd__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql deleted file mode 100644 index 3b8d494b22b5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "integrationtests".test_normalization."exchange_rate" - where (_airbyte_ab_id) in ( - select (_airbyte_ab_id) - from "exchange_rate__dbt_tmp" - ); - - insert into "integrationtests".test_normalization."exchange_rate" ("currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id") - ( - select "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id" - from "exchange_rate__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql deleted file mode 100644 index 8d6374d4880e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ /dev/null @@ -1,14 +0,0 @@ - - delete - from "integrationtests".test_normalization."renamed_dedup_cdc_excluded" - where (_airbyte_unique_key) in ( - select (_airbyte_unique_key) - from "renamed_dedup_cdc_excluded__dbt_tmp" - ); - - insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") - ( - select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" - from "renamed_dedup_cdc_excluded__dbt_tmp" - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..5afae5130fb4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,89 @@ + + + create table + "integrationtests".test_normalization."exchange_rate__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, + case when json_extract_path_text(_airbyte_data, 'new_column', true) != '' then json_extract_path_text(_airbyte_data, 'new_column', true) end as new_column, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, + case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", + case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, + case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, + case when json_extract_path_text(_airbyte_data, 'column`_''with"_quotes', true) != '' then json_extract_path_text(_airbyte_data, 'column`_''with"_quotes', true) end as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + float +) as id, + cast(currency as varchar) as currency, + cast(new_column as + float +) as new_column, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("hkd@spéçiäl & characters" as + float +) as "hkd@spéçiäl & characters", + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + cast("column`_'with""_quotes" as varchar) as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__exchange_rate_ab1 +-- exchange_rate +where 1 = 1 +), __dbt__cte__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(new_column as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') as varchar)) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__cte__exchange_rate_ab2 tmp +-- exchange_rate +where 1 = 1 +)-- Final base SQL model +select + id, + currency, + new_column, + date, + timestamp_col, + "hkd@spéçiäl & characters", + nzd, + usd, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from __dbt__cte__exchange_rate_ab3 +-- exchange_rate from "integrationtests".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 0c4e15f9ecee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,53 +0,0 @@ - - - create view "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( - -with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias --- renamed_dedup_cdc_excluded -where 1 = 1 - -), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as varchar) as name, - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__renamed_dedup_cdc_excluded_ab1 --- renamed_dedup_cdc_excluded -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_renamed_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp --- renamed_dedup_cdc_excluded -where 1 = 1 - - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml new file mode 100755 index 000000000000..9ad815875900 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql deleted file mode 100644 index e18e6a208a2c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - get_path(parse_json(_airbyte_data), '"conflict_stream_array"') as CONFLICT_STREAM_ARRAY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY as table_alias --- CONFLICT_STREAM_ARRAY -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql deleted file mode 100644 index 97e976fd6462..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - CONFLICT_STREAM_ARRAY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" --- CONFLICT_STREAM_ARRAY -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql deleted file mode 100644 index 66a710356b32..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_ARRAY as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" tmp --- CONFLICT_STREAM_ARRAY -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index 1a41bd6a787d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - - get_path(parse_json(table_alias._airbyte_data), '"conflict_stream_name"') - as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME as table_alias --- CONFLICT_STREAM_NAME -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index bf735e634dcb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(CONFLICT_STREAM_NAME as - variant -) as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" --- CONFLICT_STREAM_NAME -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index 3fe0514dbf9f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" tmp --- CONFLICT_STREAM_NAME -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index e3ef93a18792..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - - get_path(parse_json(table_alias.CONFLICT_STREAM_NAME), '"conflict_stream_name"') - as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as table_alias --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name -where 1 = 1 -and CONFLICT_STREAM_NAME is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index e9b6b9804fd9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - cast(CONFLICT_STREAM_NAME as - variant -) as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index 0616d66795ea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_HASHID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index 3e793970e09c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - to_varchar(get_path(parse_json(CONFLICT_STREAM_NAME), '"groups"')) as GROUPS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as table_alias --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and CONFLICT_STREAM_NAME is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index a405f7059b95..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - cast(GROUPS as - varchar -) as GROUPS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index 26ab7e066acf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID as - varchar -), '') || '-' || coalesce(cast(GROUPS as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql deleted file mode 100644 index d61b09ff806a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"conflict_stream_scalar"')) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR as table_alias --- CONFLICT_STREAM_SCALAR -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql deleted file mode 100644 index 703883993977..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(CONFLICT_STREAM_SCALAR as - bigint -) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" --- CONFLICT_STREAM_SCALAR -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql deleted file mode 100644 index 00e7c7886c89..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_SCALAR as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" tmp --- CONFLICT_STREAM_SCALAR -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index 6e7e79743d05..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - - get_path(parse_json(table_alias._airbyte_data), '"partition"') - as PARTITION, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES as table_alias --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index eb9c920cc525..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,22 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(DATE as - varchar -) as DATE, - cast(PARTITION as - variant -) as PARTITION, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql deleted file mode 100644 index 7b11ebe9ba64..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - get_path(parse_json(PARTITION), '"double_array_data"') as DOUBLE_ARRAY_DATA, - get_path(parse_json(PARTITION), '"DATA"') as DATA, - get_path(parse_json(PARTITION), '"column`_''with""_quotes"') as "column`_'with""_quotes", - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as table_alias --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and PARTITION is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql deleted file mode 100644 index 8c9edac47baf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - DOUBLE_ARRAY_DATA, - DATA, - "column`_'with""_quotes", - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql deleted file mode 100644 index d659cf82cab8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID as - varchar -), '') || '-' || coalesce(cast(DOUBLE_ARRAY_DATA as - varchar -), '') || '-' || coalesce(cast(DATA as - varchar -), '') || '-' || coalesce(cast("column`_'with""_quotes" as - varchar -), '') as - varchar -)) as _AIRBYTE_PARTITION_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" tmp --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql deleted file mode 100644 index 17a36065ce62..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_PARTITION_HASHID, - to_varchar(get_path(parse_json("column`_'with""_quotes".value), '"currency"')) as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -cross join table(flatten("column`_'with""_quotes")) as "column`_'with""_quotes" -where 1 = 1 -and "column`_'with""_quotes" is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql deleted file mode 100644 index de013bb4fb92..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(CURRENCY as - varchar -) as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql deleted file mode 100644 index 1996258464ce..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as - varchar -), '') || '-' || coalesce(cast(CURRENCY as - varchar -), '') as - varchar -)) as _AIRBYTE_COLUMN___WITH__QUOTES_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" tmp --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql deleted file mode 100644 index fbd05545c768..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_PARTITION_HASHID, - to_varchar(get_path(parse_json(DATA.value), '"currency"')) as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -cross join table(flatten(DATA)) as DATA -where 1 = 1 -and DATA is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql deleted file mode 100644 index bdf9cd2b386c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(CURRENCY as - varchar -) as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql deleted file mode 100644 index c4a6deea96b7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as - varchar -), '') || '-' || coalesce(cast(CURRENCY as - varchar -), '') as - varchar -)) as _AIRBYTE_DATA_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql deleted file mode 100644 index 3013309f1f90..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_PARTITION_HASHID, - to_varchar(get_path(parse_json(DOUBLE_ARRAY_DATA.value), '"id"')) as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -cross join table(flatten(DOUBLE_ARRAY_DATA)) as DOUBLE_ARRAY_DATA -where 1 = 1 -and DOUBLE_ARRAY_DATA is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql deleted file mode 100644 index 5c1e8b226a6a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(ID as - varchar -) as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql deleted file mode 100644 index 662055a5ee44..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as - varchar -), '') || '-' || coalesce(cast(ID as - varchar -), '') as - varchar -)) as _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" tmp --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index 44ffce39889f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index 437075adbab4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(DATE as - varchar -) as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql deleted file mode 100644 index 70f3a3546307..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') as - varchar -)) as _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql deleted file mode 100644 index 3c6d0a2e24d2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - get_path(parse_json(_airbyte_data), '"children"') as CHILDREN, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS as table_alias --- UNNEST_ALIAS -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql deleted file mode 100644 index 520bc1d1d7ec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - CHILDREN, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" --- UNNEST_ALIAS -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql deleted file mode 100644 index 753d624ac210..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CHILDREN as - varchar -), '') as - varchar -)) as _AIRBYTE_UNNEST_ALIAS_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" tmp --- UNNEST_ALIAS -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql deleted file mode 100644 index a4717707483e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - to_varchar(get_path(parse_json(CHILDREN.value), '"ab_id"')) as AB_ID, - - get_path(parse_json(CHILDREN.value), '"owner"') - as OWNER, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as table_alias --- CHILDREN at unnest_alias/children -cross join table(flatten(CHILDREN)) as CHILDREN -where 1 = 1 -and CHILDREN is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql deleted file mode 100644 index b0fd97c240a0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - cast(AB_ID as - bigint -) as AB_ID, - cast(OWNER as - variant -) as OWNER, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" --- CHILDREN at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql deleted file mode 100644 index 88b1593cc4d0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_UNNEST_ALIAS_HASHID as - varchar -), '') || '-' || coalesce(cast(AB_ID as - varchar -), '') || '-' || coalesce(cast(OWNER as - varchar -), '') as - varchar -)) as _AIRBYTE_CHILDREN_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" tmp --- CHILDREN at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql deleted file mode 100644 index 806d713ba2f3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CHILDREN_HASHID, - to_varchar(get_path(parse_json(OWNER), '"owner_id"')) as OWNER_ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as table_alias --- OWNER at unnest_alias/children/owner -where 1 = 1 -and OWNER is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql deleted file mode 100644 index 4e3f70fc2e82..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CHILDREN_HASHID, - cast(OWNER_ID as - bigint -) as OWNER_ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" --- OWNER at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql deleted file mode 100644 index 79189e35ffbe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_CHILDREN_HASHID as - varchar -), '') || '-' || coalesce(cast(OWNER_ID as - varchar -), '') as - varchar -)) as _AIRBYTE_OWNER_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" tmp --- OWNER at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index dc9d26c10728..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index ac66f7e51acd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(DATE as - varchar -) as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql deleted file mode 100644 index 0becb20f6e1b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') as - varchar -)) as _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql new file mode 100644 index 000000000000..9e1f9be9ff55 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql @@ -0,0 +1,68 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as + (select * from( + +with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + get_path(parse_json(PARTITION), '"double_array_data"') as DOUBLE_ARRAY_DATA, + get_path(parse_json(PARTITION), '"DATA"') as DATA, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as table_alias +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and PARTITION is not null + +), __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + DOUBLE_ARRAY_DATA, + DATA, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1 +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +), __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID as + varchar +), '') || '-' || coalesce(cast(DOUBLE_ARRAY_DATA as + varchar +), '') || '-' || coalesce(cast(DATA as + varchar +), '') as + varchar +)) as _AIRBYTE_PARTITION_HASHID, + tmp.* +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2 tmp +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + +)-- Final base SQL model +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + DOUBLE_ARRAY_DATA, + DATA, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_PARTITION_HASHID +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3 +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" +where 1 = 1 + + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql new file mode 100644 index 000000000000..3849a3fb1ea3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql @@ -0,0 +1,67 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as + (select * from( + +with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json(DATA.value), '"currency"')) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join table(flatten(DATA)) as DATA +where 1 = 1 +and DATA is not null + +), __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(CURRENCY as + varchar +) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +), __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') as + varchar +)) as _AIRBYTE_DATA_HASHID, + tmp.* +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2 tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + +)-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DATA_HASHID +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql new file mode 100644 index 000000000000..61d85bccbec4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql @@ -0,0 +1,67 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as + (select * from( + +with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json(DOUBLE_ARRAY_DATA.value), '"id"')) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join table(flatten(DOUBLE_ARRAY_DATA)) as DOUBLE_ARRAY_DATA +where 1 = 1 +and DOUBLE_ARRAY_DATA is not null + +), __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(ID as + varchar +) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1 +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +), __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID, + tmp.* +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2 tmp +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + +)-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3 +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql deleted file mode 100644 index 49df6fc5d79b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as - (select * from( - --- Final base SQL model -select - ID, - CHILDREN, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_UNNEST_ALIAS_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB3" --- UNNEST_ALIAS from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS -where 1 = 1 - - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql index a3f67db69287..b59475459004 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -30,13 +30,13 @@ scd_data as ( DATE desc, _AIRBYTE_EMITTED_AT desc ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( + case when row_number() over ( partition by ID order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + ) = 1 then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID @@ -76,6 +76,6 @@ select convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID from dedup_data where _AIRBYTE_ROW_NUM = 1 - ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY_SCD, _AIRBYTE_EMITTED_AT) ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY_SCD, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql deleted file mode 100644 index a366f2bba8cc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" as - (select * from( - --- Final base SQL model -select - ID, - CONFLICT_STREAM_ARRAY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" --- CONFLICT_STREAM_ARRAY from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql deleted file mode 100644 index a20080a7f7f4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as - (select * from( - --- Final base SQL model -select - ID, - CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" --- CONFLICT_STREAM_NAME from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql deleted file mode 100644 index 4d69fa730172..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql deleted file mode 100644 index 97d25955385f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - GROUPS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql deleted file mode 100644 index efc7cbe9dce6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" as - (select * from( - --- Final base SQL model -select - ID, - CONFLICT_STREAM_SCALAR, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" --- CONFLICT_STREAM_SCALAR from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql deleted file mode 100644 index 75228a851232..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - DOUBLE_ARRAY_DATA, - DATA, - "column`_'with""_quotes", - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_PARTITION_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql deleted file mode 100644 index 1f2adeefe899..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_COLUMN___WITH__QUOTES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql deleted file mode 100644 index cfc1ce750875..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_DATA_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql deleted file mode 100644 index 4bdef077e4b3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql deleted file mode 100644 index a470f14eceac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" as - (select * from( - --- Final base SQL model -select - ID, - DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql deleted file mode 100644 index 0657b14de313..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - AB_ID, - OWNER, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CHILDREN_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" --- CHILDREN at unnest_alias/children from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql deleted file mode 100644 index 652a9bd48931..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_CHILDREN_HASHID, - OWNER_ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_OWNER_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" --- OWNER at unnest_alias/children/owner from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql deleted file mode 100644 index 057fea2c9ee7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" as - (select * from( - --- Final base SQL model -select - ID, - DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql index b224c3ada40e..43747894b413 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -1,7 +1,43 @@ create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3" as ( --- SQL model to build a hash column based on the values of this record +with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + + get_path(parse_json(table_alias._airbyte_data), '"partition"') + as PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES as table_alias +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + +), __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + cast(PARTITION as + variant +) as PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1 +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select md5(cast(coalesce(cast(ID as varchar @@ -13,7 +49,7 @@ select varchar )) as _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" tmp +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2 tmp -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql deleted file mode 100644 index 991a40b91868..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, - {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as CONFLICT_STREAM_ARRAY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_ARRAY') }} as table_alias --- CONFLICT_STREAM_ARRAY -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql deleted file mode 100644 index 645bc6ce95f4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as {{ dbt_utils.type_string() }}) as ID, - CONFLICT_STREAM_ARRAY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('CONFLICT_STREAM_ARRAY_AB1') }} --- CONFLICT_STREAM_ARRAY -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql deleted file mode 100644 index 83752bf1352d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - array_to_string('CONFLICT_STREAM_ARRAY'), - ]) }} as _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID, - tmp.* -from {{ ref('CONFLICT_STREAM_ARRAY_AB2') }} tmp --- CONFLICT_STREAM_ARRAY -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index 5822167058d6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, - {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_NAME') }} as table_alias --- CONFLICT_STREAM_NAME -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index 73e500cda859..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as {{ dbt_utils.type_string() }}) as ID, - cast(CONFLICT_STREAM_NAME as {{ type_json() }}) as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('CONFLICT_STREAM_NAME_AB1') }} --- CONFLICT_STREAM_NAME -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index 516db74e7a46..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'CONFLICT_STREAM_NAME', - ]) }} as _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - tmp.* -from {{ ref('CONFLICT_STREAM_NAME_AB2') }} tmp --- CONFLICT_STREAM_NAME -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index 8c5659840ae3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - {{ json_extract('table_alias', 'CONFLICT_STREAM_NAME', ['conflict_stream_name'], ['conflict_stream_name']) }} as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('CONFLICT_STREAM_NAME') }} as table_alias --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name -where 1 = 1 -and CONFLICT_STREAM_NAME is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index e23598a11d34..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - cast(CONFLICT_STREAM_NAME as {{ type_json() }}) as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1') }} --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index 97ca26979ab2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_CONFLICT_STREAM_NAME_HASHID', - 'CONFLICT_STREAM_NAME', - ]) }} as _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - tmp.* -from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2') }} tmp --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index df7f67fc3449..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - {{ json_extract_scalar('CONFLICT_STREAM_NAME', ['groups'], ['groups']) }} as GROUPS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME') }} as table_alias --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and CONFLICT_STREAM_NAME is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index 1956ff59d185..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - cast(GROUPS as {{ dbt_utils.type_string() }}) as GROUPS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1') }} --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index e4a14d193cdc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID', - 'GROUPS', - ]) }} as _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID, - tmp.* -from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2') }} tmp --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql deleted file mode 100644 index 1452594dfb23..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, - {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as CONFLICT_STREAM_SCALAR, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_SCALAR') }} as table_alias --- CONFLICT_STREAM_SCALAR -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql deleted file mode 100644 index d889d1f350ce..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as {{ dbt_utils.type_string() }}) as ID, - cast(CONFLICT_STREAM_SCALAR as {{ dbt_utils.type_bigint() }}) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('CONFLICT_STREAM_SCALAR_AB1') }} --- CONFLICT_STREAM_SCALAR -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql deleted file mode 100644 index 5b6d06708f98..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'CONFLICT_STREAM_SCALAR', - ]) }} as _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID, - tmp.* -from {{ ref('CONFLICT_STREAM_SCALAR_AB2') }} tmp --- CONFLICT_STREAM_SCALAR -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql index b74a533e30d4..4c8bde0c6ffb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -1,6 +1,6 @@ {{ config( cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + unique_key = '_AIRBYTE_AB_ID', schema = "_AIRBYTE_TEST_NORMALIZATION", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql index 6acd4a631460..bdcb26988c6c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -1,6 +1,6 @@ {{ config( cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + unique_key = '_AIRBYTE_AB_ID', schema = "_AIRBYTE_TEST_NORMALIZATION", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql index d9183add3292..35e0cfd5b169 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql @@ -1,6 +1,6 @@ {{ config( cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + unique_key = '_AIRBYTE_AB_ID', schema = "_AIRBYTE_TEST_NORMALIZATION", tags = [ "nested-intermediate" ] ) }} @@ -9,12 +9,12 @@ select _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, {{ json_extract_array('PARTITION', ['double_array_data'], ['double_array_data']) }} as DOUBLE_ARRAY_DATA, {{ json_extract_array('PARTITION', ['DATA'], ['DATA']) }} as DATA, - {{ json_extract_array('PARTITION', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} as table_alias +from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD') }} as table_alias -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 and PARTITION is not null +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql deleted file mode 100644 index f610a8eb8bca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - DOUBLE_ARRAY_DATA, - DATA, - {{ adapter.quote('column`_\'with""_quotes') }}, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1') }} --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql deleted file mode 100644 index 8688fa6761c6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID', - array_to_string('DOUBLE_ARRAY_DATA'), - array_to_string('DATA'), - array_to_string(adapter.quote('column`_\'with""_quotes')), - ]) }} as _AIRBYTE_PARTITION_HASHID, - tmp.* -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2') }} tmp --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql deleted file mode 100644 index 6a50aee02e79..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION', 'PARTITION', adapter.quote('column`_\'with""_quotes')) }} -select - _AIRBYTE_PARTITION_HASHID, - {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} as table_alias --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -{{ cross_join_unnest('PARTITION', adapter.quote('column`_\'with""_quotes')) }} -where 1 = 1 -and {{ adapter.quote('column`_\'with""_quotes') }} is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql deleted file mode 100644 index 301b85b8e983..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(CURRENCY as {{ dbt_utils.type_string() }}) as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1') }} --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql deleted file mode 100644 index b4f7cd0bdb61..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_PARTITION_HASHID', - 'CURRENCY', - ]) }} as _AIRBYTE_COLUMN___WITH__QUOTES_HASHID, - tmp.* -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2') }} tmp --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql index daa25f87285e..2a03e54e6453 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql @@ -1,6 +1,5 @@ {{ config( cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), schema = "_AIRBYTE_TEST_NORMALIZATION", tags = [ "nested-intermediate" ] ) }} @@ -17,4 +16,5 @@ from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTIT {{ cross_join_unnest('PARTITION', 'DATA') }} where 1 = 1 and DATA is not null +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql deleted file mode 100644 index 81f8c127d424..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(CURRENCY as {{ dbt_utils.type_string() }}) as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1') }} --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql deleted file mode 100644 index 11dd0dc3fc02..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_PARTITION_HASHID', - 'CURRENCY', - ]) }} as _AIRBYTE_DATA_HASHID, - tmp.* -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2') }} tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql index 281b8400bb15..06494755f1af 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql @@ -1,6 +1,5 @@ {{ config( cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), schema = "_AIRBYTE_TEST_NORMALIZATION", tags = [ "nested-intermediate" ] ) }} @@ -17,4 +16,5 @@ from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTIT {{ cross_join_unnest('PARTITION', 'DOUBLE_ARRAY_DATA') }} where 1 = 1 and DOUBLE_ARRAY_DATA is not null +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql deleted file mode 100644 index f036c8235e70..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(ID as {{ dbt_utils.type_string() }}) as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1') }} --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql deleted file mode 100644 index 143f944237e4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_PARTITION_HASHID', - 'ID', - ]) }} as _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID, - tmp.* -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2') }} tmp --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index 6692c6fcf348..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} as table_alias --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index b3926083d3ef..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as {{ dbt_utils.type_string() }}) as ID, - cast(DATE as {{ dbt_utils.type_string() }}) as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1') }} --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql deleted file mode 100644 index bd557dcba2d1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'DATE', - ]) }} as _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, - tmp.* -from {{ ref('NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2') }} tmp --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql deleted file mode 100644 index f410de949a5f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, - {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as CHILDREN, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_UNNEST_ALIAS') }} as table_alias --- UNNEST_ALIAS -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql deleted file mode 100644 index 15ef88171eae..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as {{ dbt_utils.type_bigint() }}) as ID, - CHILDREN, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('UNNEST_ALIAS_AB1') }} --- UNNEST_ALIAS -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql deleted file mode 100644 index 8b4fd76705ce..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - array_to_string('CHILDREN'), - ]) }} as _AIRBYTE_UNNEST_ALIAS_HASHID, - tmp.* -from {{ ref('UNNEST_ALIAS_AB2') }} tmp --- UNNEST_ALIAS -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql deleted file mode 100644 index 3927a7ba55db..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -{{ unnest_cte('UNNEST_ALIAS', 'UNNEST_ALIAS', 'CHILDREN') }} -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - {{ json_extract_scalar(unnested_column_value('CHILDREN'), ['ab_id'], ['ab_id']) }} as AB_ID, - {{ json_extract('', unnested_column_value('CHILDREN'), ['owner'], ['owner']) }} as OWNER, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('UNNEST_ALIAS') }} as table_alias --- CHILDREN at unnest_alias/children -{{ cross_join_unnest('UNNEST_ALIAS', 'CHILDREN') }} -where 1 = 1 -and CHILDREN is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql deleted file mode 100644 index f4726f1b470e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - cast(AB_ID as {{ dbt_utils.type_bigint() }}) as AB_ID, - cast(OWNER as {{ type_json() }}) as OWNER, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('UNNEST_ALIAS_CHILDREN_AB1') }} --- CHILDREN at unnest_alias/children -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql deleted file mode 100644 index 57b65d2b40db..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_UNNEST_ALIAS_HASHID', - 'AB_ID', - 'OWNER', - ]) }} as _AIRBYTE_CHILDREN_HASHID, - tmp.* -from {{ ref('UNNEST_ALIAS_CHILDREN_AB2') }} tmp --- CHILDREN at unnest_alias/children -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql deleted file mode 100644 index a80f261e0935..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CHILDREN_HASHID, - {{ json_extract_scalar('OWNER', ['owner_id'], ['owner_id']) }} as OWNER_ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('UNNEST_ALIAS_CHILDREN') }} as table_alias --- OWNER at unnest_alias/children/owner -where 1 = 1 -and OWNER is not null - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql deleted file mode 100644 index 156bdcb32bce..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CHILDREN_HASHID, - cast(OWNER_ID as {{ dbt_utils.type_bigint() }}) as OWNER_ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('UNNEST_ALIAS_CHILDREN_OWNER_AB1') }} --- OWNER at unnest_alias/children/owner -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql deleted file mode 100644 index 07a3fe2b02c4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "nested-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_CHILDREN_HASHID', - 'OWNER_ID', - ]) }} as _AIRBYTE_OWNER_HASHID, - tmp.* -from {{ ref('UNNEST_ALIAS_CHILDREN_OWNER_AB2') }} tmp --- OWNER at unnest_alias/children/owner -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index 1c660548d9ef..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ source('TEST_NORMALIZATION_NAMESPACE', '_AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} as table_alias --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index d4ba14c99afb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as {{ dbt_utils.type_string() }}) as ID, - cast(DATE as {{ dbt_utils.type_string() }}) as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1') }} --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql deleted file mode 100644 index 56780a4b7cf3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'DATE', - ]) }} as _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, - tmp.* -from {{ ref('SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2') }} tmp --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql new file mode 100644 index 000000000000..b7009f48fa60 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql @@ -0,0 +1,20 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = '_AIRBYTE_AB_ID', + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + DOUBLE_ARRAY_DATA, + DATA, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_PARTITION_HASHID +from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3') }} +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD') }} +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql new file mode 100644 index 000000000000..216cd92f80b3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql @@ -0,0 +1,18 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DATA_HASHID +from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3') }} +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql new file mode 100644 index 000000000000..4e5d38cec2cf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql @@ -0,0 +1,18 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID +from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3') }} +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql deleted file mode 100644 index eaaa8ed6bc11..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - ID, - CHILDREN, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_UNNEST_ALIAS_HASHID -from {{ ref('UNNEST_ALIAS_AB3') }} --- UNNEST_ALIAS from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_UNNEST_ALIAS') }} -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql index a47501b4ad35..fe9543ff88ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -1,5 +1,5 @@ {{ config( - cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY_SCD", "_AIRBYTE_EMITTED_AT"], unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", schema = "TEST_NORMALIZATION", tags = [ "top-level" ] @@ -63,13 +63,13 @@ scd_data as ( DATE desc, _AIRBYTE_EMITTED_AT desc ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( + case when row_number() over ( partition by ID order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + ) = 1 then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql deleted file mode 100644 index 9f76e65e0fcd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - ID, - CONFLICT_STREAM_ARRAY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID -from {{ ref('CONFLICT_STREAM_ARRAY_AB3') }} --- CONFLICT_STREAM_ARRAY from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_ARRAY') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql deleted file mode 100644 index 713b00119ec0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - ID, - CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID -from {{ ref('CONFLICT_STREAM_NAME_AB3') }} --- CONFLICT_STREAM_NAME from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_NAME') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql deleted file mode 100644 index b28901245add..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID -from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3') }} --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name from {{ ref('CONFLICT_STREAM_NAME') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql deleted file mode 100644 index 9e873bdf985d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - GROUPS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID -from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3') }} --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql deleted file mode 100644 index 57ea8746c22d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - ID, - CONFLICT_STREAM_SCALAR, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID -from {{ ref('CONFLICT_STREAM_SCALAR_AB3') }} --- CONFLICT_STREAM_SCALAR from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_SCALAR') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql deleted file mode 100644 index 05cd2e777323..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - DOUBLE_ARRAY_DATA, - DATA, - {{ adapter.quote('column`_\'with""_quotes') }}, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_PARTITION_HASHID -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3') }} --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql deleted file mode 100644 index cfcb9c78f631..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_COLUMN___WITH__QUOTES_HASHID -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3') }} --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql deleted file mode 100644 index fb8b719df25e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_DATA_HASHID -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3') }} --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql deleted file mode 100644 index 4282011cdce6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3') }} --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql deleted file mode 100644 index e8419fbe61a8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - ID, - DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID -from {{ ref('NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3') }} --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql deleted file mode 100644 index d94783e6631d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - AB_ID, - OWNER, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CHILDREN_HASHID -from {{ ref('UNNEST_ALIAS_CHILDREN_AB3') }} --- CHILDREN at unnest_alias/children from {{ ref('UNNEST_ALIAS') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql deleted file mode 100644 index 639a820f74be..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "nested" ] -) }} --- Final base SQL model -select - _AIRBYTE_CHILDREN_HASHID, - OWNER_ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_OWNER_HASHID -from {{ ref('UNNEST_ALIAS_CHILDREN_OWNER_AB3') }} --- OWNER at unnest_alias/children/owner from {{ ref('UNNEST_ALIAS_CHILDREN') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql deleted file mode 100644 index 67e8f286cd08..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION_NAMESPACE", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - ID, - DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID -from {{ ref('SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3') }} --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION_NAMESPACE', '_AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql index adb912d6a122..ea2ec0819f5c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -1,6 +1,6 @@ {{ config( cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + unique_key = '_AIRBYTE_AB_ID', schema = "_AIRBYTE_TEST_NORMALIZATION", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql deleted file mode 100644 index e18e6a208a2c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - get_path(parse_json(_airbyte_data), '"conflict_stream_array"') as CONFLICT_STREAM_ARRAY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY as table_alias --- CONFLICT_STREAM_ARRAY -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql deleted file mode 100644 index 97e976fd6462..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - CONFLICT_STREAM_ARRAY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" --- CONFLICT_STREAM_ARRAY -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql deleted file mode 100644 index 66a710356b32..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_ARRAY as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" tmp --- CONFLICT_STREAM_ARRAY -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index 1a41bd6a787d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - - get_path(parse_json(table_alias._airbyte_data), '"conflict_stream_name"') - as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME as table_alias --- CONFLICT_STREAM_NAME -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index bf735e634dcb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(CONFLICT_STREAM_NAME as - variant -) as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" --- CONFLICT_STREAM_NAME -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index 3fe0514dbf9f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" tmp --- CONFLICT_STREAM_NAME -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index e3ef93a18792..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - - get_path(parse_json(table_alias.CONFLICT_STREAM_NAME), '"conflict_stream_name"') - as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as table_alias --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name -where 1 = 1 -and CONFLICT_STREAM_NAME is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index e9b6b9804fd9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - cast(CONFLICT_STREAM_NAME as - variant -) as CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index 0616d66795ea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_HASHID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql deleted file mode 100644 index 3e793970e09c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - to_varchar(get_path(parse_json(CONFLICT_STREAM_NAME), '"groups"')) as GROUPS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as table_alias --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 -and CONFLICT_STREAM_NAME is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql deleted file mode 100644 index a405f7059b95..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - cast(GROUPS as - varchar -) as GROUPS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql deleted file mode 100644 index 26ab7e066acf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID as - varchar -), '') || '-' || coalesce(cast(GROUPS as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql deleted file mode 100644 index d61b09ff806a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"conflict_stream_scalar"')) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR as table_alias --- CONFLICT_STREAM_SCALAR -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql deleted file mode 100644 index 703883993977..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(CONFLICT_STREAM_SCALAR as - bigint -) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" --- CONFLICT_STREAM_SCALAR -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql deleted file mode 100644 index 00e7c7886c89..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CONFLICT_STREAM_SCALAR as - varchar -), '') as - varchar -)) as _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" tmp --- CONFLICT_STREAM_SCALAR -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index 6e7e79743d05..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - - get_path(parse_json(table_alias._airbyte_data), '"partition"') - as PARTITION, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES as table_alias --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index eb9c920cc525..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,22 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(DATE as - varchar -) as DATE, - cast(PARTITION as - variant -) as PARTITION, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql deleted file mode 100644 index 7b11ebe9ba64..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - get_path(parse_json(PARTITION), '"double_array_data"') as DOUBLE_ARRAY_DATA, - get_path(parse_json(PARTITION), '"DATA"') as DATA, - get_path(parse_json(PARTITION), '"column`_''with""_quotes"') as "column`_'with""_quotes", - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as table_alias --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and PARTITION is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql deleted file mode 100644 index 8c9edac47baf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - DOUBLE_ARRAY_DATA, - DATA, - "column`_'with""_quotes", - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql deleted file mode 100644 index d659cf82cab8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID as - varchar -), '') || '-' || coalesce(cast(DOUBLE_ARRAY_DATA as - varchar -), '') || '-' || coalesce(cast(DATA as - varchar -), '') || '-' || coalesce(cast("column`_'with""_quotes" as - varchar -), '') as - varchar -)) as _AIRBYTE_PARTITION_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" tmp --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql deleted file mode 100644 index 17a36065ce62..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_PARTITION_HASHID, - to_varchar(get_path(parse_json("column`_'with""_quotes".value), '"currency"')) as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -cross join table(flatten("column`_'with""_quotes")) as "column`_'with""_quotes" -where 1 = 1 -and "column`_'with""_quotes" is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql deleted file mode 100644 index de013bb4fb92..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(CURRENCY as - varchar -) as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql deleted file mode 100644 index 1996258464ce..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as - varchar -), '') || '-' || coalesce(cast(CURRENCY as - varchar -), '') as - varchar -)) as _AIRBYTE_COLUMN___WITH__QUOTES_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" tmp --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql deleted file mode 100644 index fbd05545c768..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_PARTITION_HASHID, - to_varchar(get_path(parse_json(DATA.value), '"currency"')) as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -cross join table(flatten(DATA)) as DATA -where 1 = 1 -and DATA is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql deleted file mode 100644 index bdf9cd2b386c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(CURRENCY as - varchar -) as CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql deleted file mode 100644 index c4a6deea96b7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as - varchar -), '') || '-' || coalesce(cast(CURRENCY as - varchar -), '') as - varchar -)) as _AIRBYTE_DATA_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql deleted file mode 100644 index 3013309f1f90..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_PARTITION_HASHID, - to_varchar(get_path(parse_json(DOUBLE_ARRAY_DATA.value), '"id"')) as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -cross join table(flatten(DOUBLE_ARRAY_DATA)) as DOUBLE_ARRAY_DATA -where 1 = 1 -and DOUBLE_ARRAY_DATA is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql deleted file mode 100644 index 5c1e8b226a6a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_PARTITION_HASHID, - cast(ID as - varchar -) as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql deleted file mode 100644 index 662055a5ee44..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as - varchar -), '') || '-' || coalesce(cast(ID as - varchar -), '') as - varchar -)) as _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" tmp --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index 44ffce39889f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index 437075adbab4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(DATE as - varchar -) as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql deleted file mode 100644 index 70f3a3546307..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') as - varchar -)) as _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql deleted file mode 100644 index 3c6d0a2e24d2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - get_path(parse_json(_airbyte_data), '"children"') as CHILDREN, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS as table_alias --- UNNEST_ALIAS -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql deleted file mode 100644 index 520bc1d1d7ec..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - CHILDREN, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" --- UNNEST_ALIAS -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql deleted file mode 100644 index 753d624ac210..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CHILDREN as - varchar -), '') as - varchar -)) as _AIRBYTE_UNNEST_ALIAS_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" tmp --- UNNEST_ALIAS -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql deleted file mode 100644 index a4717707483e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql +++ /dev/null @@ -1,20 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema - -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - to_varchar(get_path(parse_json(CHILDREN.value), '"ab_id"')) as AB_ID, - - get_path(parse_json(CHILDREN.value), '"owner"') - as OWNER, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as table_alias --- CHILDREN at unnest_alias/children -cross join table(flatten(CHILDREN)) as CHILDREN -where 1 = 1 -and CHILDREN is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql deleted file mode 100644 index b0fd97c240a0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - cast(AB_ID as - bigint -) as AB_ID, - cast(OWNER as - variant -) as OWNER, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" --- CHILDREN at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql deleted file mode 100644 index 88b1593cc4d0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql +++ /dev/null @@ -1,19 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_UNNEST_ALIAS_HASHID as - varchar -), '') || '-' || coalesce(cast(AB_ID as - varchar -), '') || '-' || coalesce(cast(OWNER as - varchar -), '') as - varchar -)) as _AIRBYTE_CHILDREN_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" tmp --- CHILDREN at unnest_alias/children -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql deleted file mode 100644 index 806d713ba2f3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql +++ /dev/null @@ -1,15 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - _AIRBYTE_CHILDREN_HASHID, - to_varchar(get_path(parse_json(OWNER), '"owner_id"')) as OWNER_ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as table_alias --- OWNER at unnest_alias/children/owner -where 1 = 1 -and OWNER is not null - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql deleted file mode 100644 index 4e3f70fc2e82..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - _AIRBYTE_CHILDREN_HASHID, - cast(OWNER_ID as - bigint -) as OWNER_ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" --- OWNER at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql deleted file mode 100644 index 79189e35ffbe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(_AIRBYTE_CHILDREN_HASHID as - varchar -), '') || '-' || coalesce(cast(OWNER_ID as - varchar -), '') as - varchar -)) as _AIRBYTE_OWNER_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" tmp --- OWNER at unnest_alias/children/owner -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql deleted file mode 100644 index dc9d26c10728..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql deleted file mode 100644 index ac66f7e51acd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - varchar -) as ID, - cast(DATE as - varchar -) as DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql deleted file mode 100644 index 0becb20f6e1b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') as - varchar -)) as _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql new file mode 100644 index 000000000000..cac40defd973 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_AB_ID = DBT_INTERNAL_DEST._AIRBYTE_AB_ID + + + + when matched then update set + "_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID","DOUBLE_ARRAY_DATA" = DBT_INTERNAL_SOURCE."DOUBLE_ARRAY_DATA","DATA" = DBT_INTERNAL_SOURCE."DATA","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_PARTITION_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_PARTITION_HASHID" + + + when not matched then insert + ("_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID", "DOUBLE_ARRAY_DATA", "DATA", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_PARTITION_HASHID") + values + ("_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID", "DOUBLE_ARRAY_DATA", "DATA", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_PARTITION_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql new file mode 100644 index 000000000000..d8e002d6d55a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql @@ -0,0 +1,9 @@ +begin; + + + insert into "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" ("_AIRBYTE_PARTITION_HASHID", "CURRENCY", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DATA_HASHID") + ( + select "_AIRBYTE_PARTITION_HASHID", "CURRENCY", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DATA_HASHID" + from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA__dbt_tmp" + ); + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql new file mode 100644 index 000000000000..7579d150c518 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql @@ -0,0 +1,9 @@ +begin; + + + insert into "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" ("_AIRBYTE_PARTITION_HASHID", "ID", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DOUBLE_ARRAY_DATA_HASHID") + ( + select "_AIRBYTE_PARTITION_HASHID", "ID", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DOUBLE_ARRAY_DATA_HASHID" + from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA__dbt_tmp" + ); + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql deleted file mode 100644 index 6938df462140..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql +++ /dev/null @@ -1,26 +0,0 @@ -begin; - - - - - - - - merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as DBT_INTERNAL_DEST - using "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS__dbt_tmp" as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._AIRBYTE_AB_ID = DBT_INTERNAL_DEST._AIRBYTE_AB_ID - - - - when matched then update set - "ID" = DBT_INTERNAL_SOURCE."ID","CHILDREN" = DBT_INTERNAL_SOURCE."CHILDREN","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_UNNEST_ALIAS_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNNEST_ALIAS_HASHID" - - - when not matched then insert - ("ID", "CHILDREN", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_UNNEST_ALIAS_HASHID") - values - ("ID", "CHILDREN", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_UNNEST_ALIAS_HASHID") - -; - commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql deleted file mode 100644 index a366f2bba8cc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" as - (select * from( - --- Final base SQL model -select - ID, - CONFLICT_STREAM_ARRAY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" --- CONFLICT_STREAM_ARRAY from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql deleted file mode 100644 index a20080a7f7f4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as - (select * from( - --- Final base SQL model -select - ID, - CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" --- CONFLICT_STREAM_NAME from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql deleted file mode 100644 index 4d69fa730172..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, - CONFLICT_STREAM_NAME, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql deleted file mode 100644 index 97d25955385f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, - GROUPS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" --- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql deleted file mode 100644 index efc7cbe9dce6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" as - (select * from( - --- Final base SQL model -select - ID, - CONFLICT_STREAM_SCALAR, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" --- CONFLICT_STREAM_SCALAR from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql deleted file mode 100644 index 75228a851232..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, - DOUBLE_ARRAY_DATA, - DATA, - "column`_'with""_quotes", - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_PARTITION_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" --- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql deleted file mode 100644 index 1f2adeefe899..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_COLUMN___WITH__QUOTES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" --- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql deleted file mode 100644 index cfc1ce750875..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - CURRENCY, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_DATA_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql deleted file mode 100644 index 4bdef077e4b3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_PARTITION_HASHID, - ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" --- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql deleted file mode 100644 index a470f14eceac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" as - (select * from( - --- Final base SQL model -select - ID, - DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" --- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql deleted file mode 100644 index 0657b14de313..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_UNNEST_ALIAS_HASHID, - AB_ID, - OWNER, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_CHILDREN_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" --- CHILDREN at unnest_alias/children from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql deleted file mode 100644 index 652a9bd48931..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_CHILDREN_HASHID, - OWNER_ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_OWNER_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" --- OWNER at unnest_alias/children/owner from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql deleted file mode 100644 index 057fea2c9ee7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ /dev/null @@ -1,19 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" as - (select * from( - --- Final base SQL model -select - ID, - DATE, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" --- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES -where 1 = 1 - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql index b224c3ada40e..43747894b413 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -1,7 +1,43 @@ create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3" as ( --- SQL model to build a hash column based on the values of this record +with __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + + get_path(parse_json(table_alias._airbyte_data), '"partition"') + as PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES as table_alias +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + +), __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + cast(PARTITION as + variant +) as PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1 +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select md5(cast(coalesce(cast(ID as varchar @@ -13,7 +49,7 @@ select varchar )) as _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" tmp +from __dbt__cte__NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2 tmp -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml new file mode 100755 index 000000000000..9ad815875900 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml @@ -0,0 +1,63 @@ +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'airbyte_utils' +version: '1.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: 'normalize' + +# These configurations specify where dbt should look for different types of files. +# The `source-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +source-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +data-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + +quoting: + database: true +# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) +# all schemas should be unquoted + schema: false + identifier: true + +# You can define configurations for models in the `source-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! +models: + airbyte_utils: + +materialized: table + generated: + airbyte_ctes: + +tags: airbyte_internal_cte + +materialized: ephemeral + airbyte_incremental: + +tags: incremental_tables + +materialized: incremental + +on_schema_change: sync_all_columns + airbyte_tables: + +tags: normalized_tables + +materialized: table + airbyte_views: + +tags: airbyte_internal_views + +materialized: view + +dispatch: + - macro_namespace: dbt_utils + search_order: ['airbyte_utils', 'dbt_utils'] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql deleted file mode 100644 index 8d5282887fb0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED as table_alias --- DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql deleted file mode 100644 index 57a50d43e5bc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(NAME as - varchar -) as NAME, - cast(_AB_CDC_LSN as - float -) as _AB_CDC_LSN, - cast(_AB_CDC_UPDATED_AT as - float -) as _AB_CDC_UPDATED_AT, - cast(_AB_CDC_DELETED_AT as - float -) as _AB_CDC_DELETED_AT, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" --- DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql deleted file mode 100644 index ee207b768a20..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, - to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", - to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, - to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, - to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE as table_alias --- DEDUP_EXCHANGE_RATE -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql deleted file mode 100644 index 66b09ec2508e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql +++ /dev/null @@ -1,43 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(CURRENCY as - varchar -) as CURRENCY, - cast(nullif(DATE, '') as - date -) as DATE, - case - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') - when TIMESTAMP_COL = '' then NULL - else to_timestamp_tz(TIMESTAMP_COL) - end as TIMESTAMP_COL - , - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(HKD_SPECIAL___CHARACTERS as - varchar -) as HKD_SPECIAL___CHARACTERS, - cast(NZD as - float -) as NZD, - cast(USD as - float -) as USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" --- DEDUP_EXCHANGE_RATE -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql deleted file mode 100644 index 33e07a844a1c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, - to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", - to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, - to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, - to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE as table_alias --- EXCHANGE_RATE -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql deleted file mode 100644 index f5ef6e473984..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql +++ /dev/null @@ -1,43 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(CURRENCY as - varchar -) as CURRENCY, - cast(nullif(DATE, '') as - date -) as DATE, - case - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') - when TIMESTAMP_COL = '' then NULL - else to_timestamp_tz(TIMESTAMP_COL) - end as TIMESTAMP_COL - , - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(HKD_SPECIAL___CHARACTERS as - varchar -) as HKD_SPECIAL___CHARACTERS, - cast(NZD as - float -) as NZD, - cast(USD as - float -) as USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" --- EXCHANGE_RATE -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql deleted file mode 100644 index c1f0840c3677..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CURRENCY as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') || '-' || coalesce(cast(TIMESTAMP_COL as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as - varchar -), '') || '-' || coalesce(cast(NZD as - varchar -), '') || '-' || coalesce(cast(USD as - varchar -), '') as - varchar -)) as _AIRBYTE_EXCHANGE_RATE_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" tmp --- EXCHANGE_RATE -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql deleted file mode 100644 index 93d03ade5063..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_log_pos"')) as _AB_CDC_LOG_POS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX as table_alias --- POS_DEDUP_CDCX -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql deleted file mode 100644 index 82b6ca1c2619..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(NAME as - varchar -) as NAME, - cast(_AB_CDC_LSN as - float -) as _AB_CDC_LSN, - cast(_AB_CDC_UPDATED_AT as - float -) as _AB_CDC_UPDATED_AT, - cast(_AB_CDC_DELETED_AT as - float -) as _AB_CDC_DELETED_AT, - cast(_AB_CDC_LOG_POS as - float -) as _AB_CDC_LOG_POS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" --- POS_DEDUP_CDCX -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql deleted file mode 100644 index 123971c4e5dc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED as table_alias --- RENAMED_DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql deleted file mode 100644 index 35438f37c247..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" --- RENAMED_DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql deleted file mode 100644 index 08e7e20215a1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_UNIQUE_KEY, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" --- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED -where 1 = 1 -and _AIRBYTE_ACTIVE_ROW = 1 - - ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql deleted file mode 100644 index 3420de9fa251..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as - (select * from( - --- Final base SQL model -select - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - "HKD@spéçiäl & characters", - HKD_SPECIAL___CHARACTERS, - NZD, - USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_EXCHANGE_RATE_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB3" --- EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE -where 1 = 1 - - ) order by (_AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql deleted file mode 100644 index f5c3fb018edc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql +++ /dev/null @@ -1,21 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_UNIQUE_KEY, - ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" --- RENAMED_DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED -where 1 = 1 -and _AIRBYTE_ACTIVE_ROW = 1 - - ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql deleted file mode 100644 index 343251eb8988..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql +++ /dev/null @@ -1,93 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" as - (select * from( - -with - -input_data as ( - select * - from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" - -- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(coalesce(cast(ID as - varchar -), '') as - varchar -)) as _AIRBYTE_UNIQUE_KEY, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as - varchar -), cast(_AB_CDC_UPDATED_AT as - varchar -) - order by _AIRBYTE_AB_ID - ) as _AIRBYTE_ROW_NUM, - md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as - varchar -), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as - varchar -), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as - varchar -), '') as - varchar -)) as _AIRBYTE_UNIQUE_KEY_SCD, - scd_data.* - from scd_data -) -select - _AIRBYTE_UNIQUE_KEY, - _AIRBYTE_UNIQUE_KEY_SCD, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_START_AT, - _AIRBYTE_END_AT, - _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from dedup_data where _AIRBYTE_ROW_NUM = 1 - ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql index f1a29aefb339..29a0be4f699d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -41,7 +41,7 @@ scd_data as ( DATE desc, _AIRBYTE_EMITTED_AT desc ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( + case when row_number() over ( partition by ID, CURRENCY, cast(NZD as varchar ) @@ -49,7 +49,7 @@ scd_data as ( DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + ) = 1 then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID @@ -94,6 +94,6 @@ select convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID from dedup_data where _AIRBYTE_ROW_NUM = 1 - ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY_SCD, _AIRBYTE_EMITTED_AT) ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY_SCD, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql deleted file mode 100644 index c171a978fe39..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql +++ /dev/null @@ -1,77 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" as - (select * from( - -with - -input_data as ( - select * - from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB3" - -- RENAMED_DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(coalesce(cast(ID as - varchar -), '') as - varchar -)) as _AIRBYTE_UNIQUE_KEY, - ID, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT - order by _AIRBYTE_AB_ID - ) as _AIRBYTE_ROW_NUM, - md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as - varchar -), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as - varchar -), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as - varchar -), '') as - varchar -)) as _AIRBYTE_UNIQUE_KEY_SCD, - scd_data.* - from scd_data -) -select - _AIRBYTE_UNIQUE_KEY, - _AIRBYTE_UNIQUE_KEY_SCD, - ID, - _AIRBYTE_START_AT, - _AIRBYTE_END_AT, - _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID -from dedup_data where _AIRBYTE_ROW_NUM = 1 - ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql new file mode 100644 index 000000000000..b75b82889b7c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -0,0 +1,117 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as + (select * from( + +with __dbt__cte__EXCHANGE_RATE_AB1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, + to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", + to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, + to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, + to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, + to_varchar(get_path(parse_json(_airbyte_data), '"column`_''with""_quotes"')) as "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE as table_alias +-- EXCHANGE_RATE +where 1 = 1 +), __dbt__cte__EXCHANGE_RATE_AB2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(CURRENCY as + varchar +) as CURRENCY, + cast(nullif(DATE, '') as + date +) as DATE, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + when TIMESTAMP_COL = '' then NULL + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(HKD_SPECIAL___CHARACTERS as + varchar +) as HKD_SPECIAL___CHARACTERS, + cast(NZD as + float +) as NZD, + cast(USD as + float +) as USD, + cast("column`_'with""_quotes" as + varchar +) as "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from __dbt__cte__EXCHANGE_RATE_AB1 +-- EXCHANGE_RATE +where 1 = 1 +), __dbt__cte__EXCHANGE_RATE_AB3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(TIMESTAMP_COL as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as + varchar +), '') || '-' || coalesce(cast(NZD as + varchar +), '') || '-' || coalesce(cast(USD as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _AIRBYTE_EXCHANGE_RATE_HASHID, + tmp.* +from __dbt__cte__EXCHANGE_RATE_AB2 tmp +-- EXCHANGE_RATE +where 1 = 1 +)-- Final base SQL model +select + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + "HKD@spéçiäl & characters", + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_EXCHANGE_RATE_HASHID +from __dbt__cte__EXCHANGE_RATE_AB3 +-- EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql deleted file mode 100644 index 198a2627ab68..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_UNIQUE_KEY, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" --- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX -where 1 = 1 -and _AIRBYTE_ACTIVE_ROW = 1 - ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql deleted file mode 100644 index 24b58ed5bf4d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql +++ /dev/null @@ -1,99 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" as - (select * from( - -with - -input_data as ( - select * - from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" - -- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(coalesce(cast(ID as - varchar -), '') as - varchar -)) as _AIRBYTE_UNIQUE_KEY, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as - varchar -), cast(_AB_CDC_UPDATED_AT as - varchar -), cast(_AB_CDC_LOG_POS as - varchar -) - order by _AIRBYTE_AB_ID - ) as _AIRBYTE_ROW_NUM, - md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as - varchar -), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as - varchar -), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as - varchar -), '') as - varchar -)) as _AIRBYTE_UNIQUE_KEY_SCD, - scd_data.* - from scd_data -) -select - _AIRBYTE_UNIQUE_KEY, - _AIRBYTE_UNIQUE_KEY_SCD, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_START_AT, - _AIRBYTE_END_AT, - _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from dedup_data where _AIRBYTE_ROW_NUM = 1 - ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql deleted file mode 100644 index 522991af25cd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(NAME as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LSN as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as - varchar -), '') as - varchar -)) as _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" tmp --- DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql index 0b361061abf3..3365a54b68b3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql @@ -1,7 +1,67 @@ create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB3" as ( --- SQL model to build a hash column based on the values of this record +with __dbt__cte__DEDUP_EXCHANGE_RATE_AB1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, + to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", + to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, + to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, + to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE as table_alias +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + +), __dbt__cte__DEDUP_EXCHANGE_RATE_AB2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(CURRENCY as + varchar +) as CURRENCY, + cast(nullif(DATE, '') as + date +) as DATE, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + when TIMESTAMP_COL = '' then NULL + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(HKD_SPECIAL___CHARACTERS as + varchar +) as HKD_SPECIAL___CHARACTERS, + cast(NZD as + float +) as NZD, + cast(USD as + float +) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from __dbt__cte__DEDUP_EXCHANGE_RATE_AB1 +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select md5(cast(coalesce(cast(ID as varchar @@ -23,7 +83,7 @@ select varchar )) as _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID, tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" tmp +from __dbt__cte__DEDUP_EXCHANGE_RATE_AB2 tmp -- DEDUP_EXCHANGE_RATE where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql deleted file mode 100644 index c1189b3c1dfd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql +++ /dev/null @@ -1,25 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(NAME as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LSN as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as - varchar -), '') as - varchar -)) as _AIRBYTE_POS_DEDUP_CDCX_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" tmp --- POS_DEDUP_CDCX -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql deleted file mode 100644 index 624b8d286061..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') as - varchar -)) as _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" tmp --- RENAMED_DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql deleted file mode 100644 index d1960bc9d6f6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, - {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as NAME, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _AB_CDC_LSN, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _AB_CDC_UPDATED_AT, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _AB_CDC_DELETED_AT, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} as table_alias --- DEDUP_CDC_EXCLUDED -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql deleted file mode 100644 index 53573e0731e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as {{ dbt_utils.type_bigint() }}) as ID, - cast(NAME as {{ dbt_utils.type_string() }}) as NAME, - cast(_AB_CDC_LSN as {{ dbt_utils.type_float() }}) as _AB_CDC_LSN, - cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_UPDATED_AT, - cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_DELETED_AT, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('DEDUP_CDC_EXCLUDED_AB1') }} --- DEDUP_CDC_EXCLUDED -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql index 52686abcae76..1baae8b96a9a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql @@ -1,6 +1,6 @@ {{ config( cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + unique_key = '_AIRBYTE_AB_ID', schema = "_AIRBYTE_TEST_NORMALIZATION", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql index 66aad2b9bd12..403aab56cd0f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql @@ -1,6 +1,6 @@ {{ config( cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + unique_key = '_AIRBYTE_AB_ID', schema = "_AIRBYTE_TEST_NORMALIZATION", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql deleted file mode 100644 index d2d59c06a80b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, - {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as CURRENCY, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as DATE, - {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as TIMESTAMP_COL, - {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as {{ adapter.quote('HKD@spéçiäl & characters') }}, - {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as HKD_SPECIAL___CHARACTERS, - {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, - {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_EXCHANGE_RATE') }} as table_alias --- EXCHANGE_RATE -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql deleted file mode 100644 index d61195d2eb84..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql +++ /dev/null @@ -1,32 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as {{ dbt_utils.type_bigint() }}) as ID, - cast(CURRENCY as {{ dbt_utils.type_string() }}) as CURRENCY, - cast({{ empty_string_to_null('DATE') }} as {{ type_date() }}) as DATE, - case - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') - when TIMESTAMP_COL = '' then NULL - else to_timestamp_tz(TIMESTAMP_COL) - end as TIMESTAMP_COL - , - cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }}, - cast(HKD_SPECIAL___CHARACTERS as {{ dbt_utils.type_string() }}) as HKD_SPECIAL___CHARACTERS, - cast(NZD as {{ dbt_utils.type_float() }}) as NZD, - cast(USD as {{ dbt_utils.type_float() }}) as USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('EXCHANGE_RATE_AB1') }} --- EXCHANGE_RATE -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql deleted file mode 100644 index db5f1117000f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'CURRENCY', - 'DATE', - 'TIMESTAMP_COL', - adapter.quote('HKD@spéçiäl & characters'), - 'HKD_SPECIAL___CHARACTERS', - 'NZD', - 'USD', - ]) }} as _AIRBYTE_EXCHANGE_RATE_HASHID, - tmp.* -from {{ ref('EXCHANGE_RATE_AB2') }} tmp --- EXCHANGE_RATE -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql deleted file mode 100644 index cae6bceaf73d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, - {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as NAME, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _AB_CDC_LSN, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _AB_CDC_UPDATED_AT, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _AB_CDC_DELETED_AT, - {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _AB_CDC_LOG_POS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} as table_alias --- POS_DEDUP_CDCX -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql deleted file mode 100644 index 9e48e834cd99..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as {{ dbt_utils.type_bigint() }}) as ID, - cast(NAME as {{ dbt_utils.type_string() }}) as NAME, - cast(_AB_CDC_LSN as {{ dbt_utils.type_float() }}) as _AB_CDC_LSN, - cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_UPDATED_AT, - cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_DELETED_AT, - cast(_AB_CDC_LOG_POS as {{ dbt_utils.type_float() }}) as _AB_CDC_LOG_POS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('POS_DEDUP_CDCX_AB1') }} --- POS_DEDUP_CDCX -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql deleted file mode 100644 index de7672d5c9df..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} as table_alias --- RENAMED_DEDUP_CDC_EXCLUDED -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql deleted file mode 100644 index 8597d9cbf0fa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as {{ dbt_utils.type_bigint() }}) as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT -from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB1') }} --- RENAMED_DEDUP_CDC_EXCLUDED -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql deleted file mode 100644 index 6aa5c273499d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], - unique_key = "_AIRBYTE_UNIQUE_KEY", - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _AIRBYTE_UNIQUE_KEY, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from {{ ref('DEDUP_CDC_EXCLUDED_SCD') }} --- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} -where 1 = 1 -and _AIRBYTE_ACTIVE_ROW = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql deleted file mode 100644 index 533791bfb80e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ /dev/null @@ -1,25 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - HKD_SPECIAL___CHARACTERS, - NZD, - USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_EXCHANGE_RATE_HASHID -from {{ ref('EXCHANGE_RATE_AB3') }} --- EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_EXCHANGE_RATE') }} -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql deleted file mode 100644 index 6c761d93c54f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], - unique_key = "_AIRBYTE_UNIQUE_KEY", - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _AIRBYTE_UNIQUE_KEY, - ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID -from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_SCD') }} --- RENAMED_DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} -where 1 = 1 -and _AIRBYTE_ACTIVE_ROW = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql deleted file mode 100644 index 6fcbf7b8e63d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql +++ /dev/null @@ -1,112 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], - unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} - -- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} - where 1 = 1 - {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} -), -new_data_ids as ( - -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'ID', - ]) }} as _AIRBYTE_UNIQUE_KEY - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('DEDUP_CDC_EXCLUDED_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} as inc_data on 1 = 0 - where _AIRBYTE_ACTIVE_ROW = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('DEDUP_CDC_EXCLUDED_AB3')) }} from new_data - union all - select {{ dbt_utils.star(ref('DEDUP_CDC_EXCLUDED_AB3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} - -- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'ID', - ]) }} as _AIRBYTE_UNIQUE_KEY, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_string() }}), cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_string() }}) - order by _AIRBYTE_AB_ID - ) as _AIRBYTE_ROW_NUM, - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_UNIQUE_KEY', - '_AIRBYTE_START_AT', - '_AIRBYTE_EMITTED_AT', '_AB_CDC_DELETED_AT', '_AB_CDC_UPDATED_AT' - ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, - scd_data.* - from scd_data -) -select - _AIRBYTE_UNIQUE_KEY, - _AIRBYTE_UNIQUE_KEY_SCD, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_START_AT, - _AIRBYTE_END_AT, - _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from dedup_data where _AIRBYTE_ROW_NUM = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql index 060a3ccb799a..7665cf7231c5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -1,5 +1,5 @@ {{ config( - cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY_SCD", "_AIRBYTE_EMITTED_AT"], unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", schema = "TEST_NORMALIZATION", tags = [ "top-level" ] @@ -72,13 +72,13 @@ scd_data as ( DATE desc, _AIRBYTE_EMITTED_AT desc ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( + case when row_number() over ( partition by ID, CURRENCY, cast(NZD as {{ dbt_utils.type_string() }}) order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + ) = 1 then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql deleted file mode 100644 index c212769bc05f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql +++ /dev/null @@ -1,104 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], - unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3') }} - -- RENAMED_DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} - where 1 = 1 - {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} -), -new_data_ids as ( - -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'ID', - ]) }} as _AIRBYTE_UNIQUE_KEY - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3') }} as inc_data on 1 = 0 - where _AIRBYTE_ACTIVE_ROW = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3')) }} from new_data - union all - select {{ dbt_utils.star(ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3') }} - -- RENAMED_DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'ID', - ]) }} as _AIRBYTE_UNIQUE_KEY, - ID, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT - order by _AIRBYTE_AB_ID - ) as _AIRBYTE_ROW_NUM, - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_UNIQUE_KEY', - '_AIRBYTE_START_AT', - '_AIRBYTE_EMITTED_AT' - ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, - scd_data.* - from scd_data -) -select - _AIRBYTE_UNIQUE_KEY, - _AIRBYTE_UNIQUE_KEY_SCD, - ID, - _AIRBYTE_START_AT, - _AIRBYTE_END_AT, - _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID -from dedup_data where _AIRBYTE_ROW_NUM = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql new file mode 100644 index 000000000000..b2a21734bf63 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -0,0 +1,25 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = '_AIRBYTE_AB_ID', + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + {{ adapter.quote('column`_\'with""_quotes') }}, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_EXCHANGE_RATE_HASHID +from {{ ref('EXCHANGE_RATE_AB3') }} +-- EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_EXCHANGE_RATE') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql deleted file mode 100644 index 1ab6634ee9b5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], - unique_key = "_AIRBYTE_UNIQUE_KEY", - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} --- Final base SQL model -select - _AIRBYTE_UNIQUE_KEY, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from {{ ref('POS_DEDUP_CDCX_SCD') }} --- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} -where 1 = 1 -and _AIRBYTE_ACTIVE_ROW = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql deleted file mode 100644 index 257fa09afe15..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql +++ /dev/null @@ -1,114 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], - unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", - schema = "TEST_NORMALIZATION", - tags = [ "top-level" ] -) }} -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('POS_DEDUP_CDCX_AB3') }} - -- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} - where 1 = 1 - {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} -), -new_data_ids as ( - -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'ID', - ]) }} as _AIRBYTE_UNIQUE_KEY - from new_data -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('POS_DEDUP_CDCX_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) - left join {{ ref('POS_DEDUP_CDCX_AB3') }} as inc_data on 1 = 0 - where _AIRBYTE_ACTIVE_ROW = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('POS_DEDUP_CDCX_AB3')) }} from new_data - union all - select {{ dbt_utils.star(ref('POS_DEDUP_CDCX_AB3')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('POS_DEDUP_CDCX_AB3') }} - -- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'ID', - ]) }} as _AIRBYTE_UNIQUE_KEY, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_string() }}), cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_string() }}), cast(_AB_CDC_LOG_POS as {{ dbt_utils.type_string() }}) - order by _AIRBYTE_AB_ID - ) as _AIRBYTE_ROW_NUM, - {{ dbt_utils.surrogate_key([ - '_AIRBYTE_UNIQUE_KEY', - '_AIRBYTE_START_AT', - '_AIRBYTE_EMITTED_AT', '_AB_CDC_DELETED_AT', '_AB_CDC_UPDATED_AT', '_AB_CDC_LOG_POS' - ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, - scd_data.* - from scd_data -) -select - _AIRBYTE_UNIQUE_KEY, - _AIRBYTE_UNIQUE_KEY_SCD, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_START_AT, - _AIRBYTE_END_AT, - _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from dedup_data where _AIRBYTE_ROW_NUM = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql deleted file mode 100644 index 1899392af6fc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'NAME', - '_AB_CDC_LSN', - '_AB_CDC_UPDATED_AT', - '_AB_CDC_DELETED_AT', - ]) }} as _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID, - tmp.* -from {{ ref('DEDUP_CDC_EXCLUDED_AB2') }} tmp --- DEDUP_CDC_EXCLUDED -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql index a78412000006..7bd3acebf826 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql @@ -1,6 +1,6 @@ {{ config( cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + unique_key = '_AIRBYTE_AB_ID', schema = "_AIRBYTE_TEST_NORMALIZATION", tags = [ "top-level-intermediate" ] ) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql deleted file mode 100644 index c886fcf1eadf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - 'NAME', - '_AB_CDC_LSN', - '_AB_CDC_UPDATED_AT', - '_AB_CDC_DELETED_AT', - '_AB_CDC_LOG_POS', - ]) }} as _AIRBYTE_POS_DEDUP_CDCX_HASHID, - tmp.* -from {{ ref('POS_DEDUP_CDCX_AB2') }} tmp --- POS_DEDUP_CDCX -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql deleted file mode 100644 index d3b3c50580d0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config( - cluster_by = ["_AIRBYTE_EMITTED_AT"], - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), - schema = "_AIRBYTE_TEST_NORMALIZATION", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'ID', - ]) }} as _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID, - tmp.* -from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB2') }} tmp --- RENAMED_DEDUP_CDC_EXCLUDED -where 1 = 1 -{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql deleted file mode 100644 index 8d5282887fb0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED as table_alias --- DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql deleted file mode 100644 index 57a50d43e5bc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql +++ /dev/null @@ -1,28 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(NAME as - varchar -) as NAME, - cast(_AB_CDC_LSN as - float -) as _AB_CDC_LSN, - cast(_AB_CDC_UPDATED_AT as - float -) as _AB_CDC_UPDATED_AT, - cast(_AB_CDC_DELETED_AT as - float -) as _AB_CDC_DELETED_AT, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" --- DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql deleted file mode 100644 index ee207b768a20..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, - to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", - to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, - to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, - to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE as table_alias --- DEDUP_EXCHANGE_RATE -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql deleted file mode 100644 index 66b09ec2508e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql +++ /dev/null @@ -1,43 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(CURRENCY as - varchar -) as CURRENCY, - cast(nullif(DATE, '') as - date -) as DATE, - case - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') - when TIMESTAMP_COL = '' then NULL - else to_timestamp_tz(TIMESTAMP_COL) - end as TIMESTAMP_COL - , - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(HKD_SPECIAL___CHARACTERS as - varchar -) as HKD_SPECIAL___CHARACTERS, - cast(NZD as - float -) as NZD, - cast(USD as - float -) as USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" --- DEDUP_EXCHANGE_RATE -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql deleted file mode 100644 index 33e07a844a1c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql +++ /dev/null @@ -1,21 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, - to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, - to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", - to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, - to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, - to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE as table_alias --- EXCHANGE_RATE -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql deleted file mode 100644 index f5ef6e473984..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql +++ /dev/null @@ -1,43 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(CURRENCY as - varchar -) as CURRENCY, - cast(nullif(DATE, '') as - date -) as DATE, - case - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') - when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') - when TIMESTAMP_COL = '' then NULL - else to_timestamp_tz(TIMESTAMP_COL) - end as TIMESTAMP_COL - , - cast("HKD@spéçiäl & characters" as - float -) as "HKD@spéçiäl & characters", - cast(HKD_SPECIAL___CHARACTERS as - varchar -) as HKD_SPECIAL___CHARACTERS, - cast(NZD as - float -) as NZD, - cast(USD as - float -) as USD, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" --- EXCHANGE_RATE -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql deleted file mode 100644 index c1f0840c3677..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(CURRENCY as - varchar -), '') || '-' || coalesce(cast(DATE as - varchar -), '') || '-' || coalesce(cast(TIMESTAMP_COL as - varchar -), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as - varchar -), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as - varchar -), '') || '-' || coalesce(cast(NZD as - varchar -), '') || '-' || coalesce(cast(USD as - varchar -), '') as - varchar -)) as _AIRBYTE_EXCHANGE_RATE_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" tmp --- EXCHANGE_RATE -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql deleted file mode 100644 index 93d03ade5063..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql +++ /dev/null @@ -1,18 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, - to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_log_pos"')) as _AB_CDC_LOG_POS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX as table_alias --- POS_DEDUP_CDCX -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql deleted file mode 100644 index 82b6ca1c2619..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql +++ /dev/null @@ -1,30 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - cast(NAME as - varchar -) as NAME, - cast(_AB_CDC_LSN as - float -) as _AB_CDC_LSN, - cast(_AB_CDC_UPDATED_AT as - float -) as _AB_CDC_UPDATED_AT, - cast(_AB_CDC_DELETED_AT as - float -) as _AB_CDC_DELETED_AT, - cast(_AB_CDC_LOG_POS as - float -) as _AB_CDC_LOG_POS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" --- POS_DEDUP_CDCX -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql deleted file mode 100644 index 123971c4e5dc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql +++ /dev/null @@ -1,14 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED as table_alias --- RENAMED_DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql deleted file mode 100644 index 35438f37c247..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(ID as - bigint -) as ID, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" --- RENAMED_DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql deleted file mode 100644 index 522240329e42..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql +++ /dev/null @@ -1,26 +0,0 @@ -begin; - - - - - - - - merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED" as DBT_INTERNAL_DEST - using "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED__dbt_tmp" as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY - - - - when matched then update set - "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","ID" = DBT_INTERNAL_SOURCE."ID","NAME" = DBT_INTERNAL_SOURCE."NAME","_AB_CDC_LSN" = DBT_INTERNAL_SOURCE."_AB_CDC_LSN","_AB_CDC_UPDATED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_UPDATED_AT","_AB_CDC_DELETED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_DELETED_AT","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" - - - when not matched then insert - ("_AIRBYTE_UNIQUE_KEY", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") - values - ("_AIRBYTE_UNIQUE_KEY", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") - -; - commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql deleted file mode 100644 index fc67dd05b1bf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ /dev/null @@ -1,26 +0,0 @@ -begin; - - - - - - - - merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as DBT_INTERNAL_DEST - using "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE__dbt_tmp" as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._AIRBYTE_AB_ID = DBT_INTERNAL_DEST._AIRBYTE_AB_ID - - - - when matched then update set - "ID" = DBT_INTERNAL_SOURCE."ID","CURRENCY" = DBT_INTERNAL_SOURCE."CURRENCY","DATE" = DBT_INTERNAL_SOURCE."DATE","TIMESTAMP_COL" = DBT_INTERNAL_SOURCE."TIMESTAMP_COL","HKD@spéçiäl & characters" = DBT_INTERNAL_SOURCE."HKD@spéçiäl & characters","HKD_SPECIAL___CHARACTERS" = DBT_INTERNAL_SOURCE."HKD_SPECIAL___CHARACTERS","NZD" = DBT_INTERNAL_SOURCE."NZD","USD" = DBT_INTERNAL_SOURCE."USD","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_EXCHANGE_RATE_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_EXCHANGE_RATE_HASHID" - - - when not matched then insert - ("ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_EXCHANGE_RATE_HASHID") - values - ("ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_EXCHANGE_RATE_HASHID") - -; - commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql deleted file mode 100644 index 3ddd6bd2a029..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql +++ /dev/null @@ -1,26 +0,0 @@ -begin; - - - - - - - - merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED" as DBT_INTERNAL_DEST - using "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED__dbt_tmp" as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY - - - - when matched then update set - "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","ID" = DBT_INTERNAL_SOURCE."ID","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" - - - when not matched then insert - ("_AIRBYTE_UNIQUE_KEY", "ID", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") - values - ("_AIRBYTE_UNIQUE_KEY", "ID", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") - -; - commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql deleted file mode 100644 index 823c37218a10..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql +++ /dev/null @@ -1,26 +0,0 @@ -begin; - - - - - - - - merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" as DBT_INTERNAL_DEST - using "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD__dbt_tmp" as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY_SCD = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY_SCD - - - - when matched then update set - "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","_AIRBYTE_UNIQUE_KEY_SCD" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY_SCD","ID" = DBT_INTERNAL_SOURCE."ID","NAME" = DBT_INTERNAL_SOURCE."NAME","_AB_CDC_LSN" = DBT_INTERNAL_SOURCE."_AB_CDC_LSN","_AB_CDC_UPDATED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_UPDATED_AT","_AB_CDC_DELETED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_DELETED_AT","_AIRBYTE_START_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_START_AT","_AIRBYTE_END_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_END_AT","_AIRBYTE_ACTIVE_ROW" = DBT_INTERNAL_SOURCE."_AIRBYTE_ACTIVE_ROW","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" - - - when not matched then insert - ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") - values - ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") - -; - commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql deleted file mode 100644 index f95adde306f8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql +++ /dev/null @@ -1,26 +0,0 @@ -begin; - - - - - - - - merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" as DBT_INTERNAL_DEST - using "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD__dbt_tmp" as DBT_INTERNAL_SOURCE - on - DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY_SCD = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY_SCD - - - - when matched then update set - "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","_AIRBYTE_UNIQUE_KEY_SCD" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY_SCD","ID" = DBT_INTERNAL_SOURCE."ID","_AIRBYTE_START_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_START_AT","_AIRBYTE_END_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_END_AT","_AIRBYTE_ACTIVE_ROW" = DBT_INTERNAL_SOURCE."_AIRBYTE_ACTIVE_ROW","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" - - - when not matched then insert - ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") - values - ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") - -; - commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql new file mode 100644 index 000000000000..b75b82889b7c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -0,0 +1,117 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as + (select * from( + +with __dbt__cte__EXCHANGE_RATE_AB1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, + to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", + to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, + to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, + to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, + to_varchar(get_path(parse_json(_airbyte_data), '"column`_''with""_quotes"')) as "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE as table_alias +-- EXCHANGE_RATE +where 1 = 1 +), __dbt__cte__EXCHANGE_RATE_AB2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(CURRENCY as + varchar +) as CURRENCY, + cast(nullif(DATE, '') as + date +) as DATE, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + when TIMESTAMP_COL = '' then NULL + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(HKD_SPECIAL___CHARACTERS as + varchar +) as HKD_SPECIAL___CHARACTERS, + cast(NZD as + float +) as NZD, + cast(USD as + float +) as USD, + cast("column`_'with""_quotes" as + varchar +) as "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from __dbt__cte__EXCHANGE_RATE_AB1 +-- EXCHANGE_RATE +where 1 = 1 +), __dbt__cte__EXCHANGE_RATE_AB3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(TIMESTAMP_COL as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as + varchar +), '') || '-' || coalesce(cast(NZD as + varchar +), '') || '-' || coalesce(cast(USD as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _AIRBYTE_EXCHANGE_RATE_HASHID, + tmp.* +from __dbt__cte__EXCHANGE_RATE_AB2 tmp +-- EXCHANGE_RATE +where 1 = 1 +)-- Final base SQL model +select + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + "HKD@spéçiäl & characters", + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_EXCHANGE_RATE_HASHID +from __dbt__cte__EXCHANGE_RATE_AB3 +-- EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql deleted file mode 100644 index 198a2627ab68..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" as - (select * from( - --- Final base SQL model -select - _AIRBYTE_UNIQUE_KEY, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" --- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX -where 1 = 1 -and _AIRBYTE_ACTIVE_ROW = 1 - ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql deleted file mode 100644 index 24b58ed5bf4d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql +++ /dev/null @@ -1,99 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" as - (select * from( - -with - -input_data as ( - select * - from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" - -- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX -), - -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - md5(cast(coalesce(cast(ID as - varchar -), '') as - varchar -)) as _AIRBYTE_UNIQUE_KEY, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by - _AIRBYTE_EMITTED_AT is null asc, - _AIRBYTE_EMITTED_AT desc, - _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as - varchar -), cast(_AB_CDC_UPDATED_AT as - varchar -), cast(_AB_CDC_LOG_POS as - varchar -) - order by _AIRBYTE_AB_ID - ) as _AIRBYTE_ROW_NUM, - md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as - varchar -), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as - varchar -), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as - varchar -), '') as - varchar -)) as _AIRBYTE_UNIQUE_KEY_SCD, - scd_data.* - from scd_data -) -select - _AIRBYTE_UNIQUE_KEY, - _AIRBYTE_UNIQUE_KEY_SCD, - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_START_AT, - _AIRBYTE_END_AT, - _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_AB_ID, - _AIRBYTE_EMITTED_AT, - convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from dedup_data where _AIRBYTE_ROW_NUM = 1 - ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) - ); - alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql deleted file mode 100644 index 522991af25cd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql +++ /dev/null @@ -1,24 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(NAME as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LSN as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as - varchar -), '') as - varchar -)) as _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" tmp --- DEDUP_CDC_EXCLUDED -where 1 = 1 - - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql index 0b361061abf3..3365a54b68b3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql @@ -1,7 +1,67 @@ create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB3" as ( --- SQL model to build a hash column based on the values of this record +with __dbt__cte__DEDUP_EXCHANGE_RATE_AB1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, + to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", + to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, + to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, + to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE as table_alias +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + +), __dbt__cte__DEDUP_EXCHANGE_RATE_AB2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(CURRENCY as + varchar +) as CURRENCY, + cast(nullif(DATE, '') as + date +) as DATE, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + when TIMESTAMP_COL = '' then NULL + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(HKD_SPECIAL___CHARACTERS as + varchar +) as HKD_SPECIAL___CHARACTERS, + cast(NZD as + float +) as NZD, + cast(USD as + float +) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from __dbt__cte__DEDUP_EXCHANGE_RATE_AB1 +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record select md5(cast(coalesce(cast(ID as varchar @@ -23,7 +83,7 @@ select varchar )) as _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID, tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" tmp +from __dbt__cte__DEDUP_EXCHANGE_RATE_AB2 tmp -- DEDUP_EXCHANGE_RATE where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql deleted file mode 100644 index c1189b3c1dfd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql +++ /dev/null @@ -1,25 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') || '-' || coalesce(cast(NAME as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LSN as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as - varchar -), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as - varchar -), '') as - varchar -)) as _AIRBYTE_POS_DEDUP_CDCX_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" tmp --- POS_DEDUP_CDCX -where 1 = 1 - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql deleted file mode 100644 index 624b8d286061..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql +++ /dev/null @@ -1,16 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB3" as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(ID as - varchar -), '') as - varchar -)) as _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" tmp --- RENAMED_DEDUP_CDC_EXCLUDED -where 1 = 1 - - );