From 34baff87b8fc66569018cb64af12710b13ece326 Mon Sep 17 00:00:00 2001 From: Christophe Duong Date: Fri, 29 Oct 2021 14:36:10 +0200 Subject: [PATCH] Regenerate Normalization Integration Tests' output SQL files --- ..._columns_resulting_into_long_names_ab4.sql | 15 -- ..._columns_resulting_into_long_names_scd.sql | 29 ---- .../conflict_stream_array_ab1.sql | 7 +- .../conflict_stream_array_ab2.sql | 7 +- .../conflict_stream_array_ab3.sql | 3 +- .../conflict_stream_name_ab1.sql | 7 +- .../conflict_stream_name_ab2.sql | 7 +- .../conflict_stream_name_ab3.sql | 3 +- ...t_stream_name_conflict_stream_name_ab1.sql | 9 +- ...t_stream_name_conflict_stream_name_ab2.sql | 7 +- ...t_stream_name_conflict_stream_name_ab3.sql | 3 +- ...t_stream_name_conflict_stream_name_ab1.sql | 9 +- ...t_stream_name_conflict_stream_name_ab2.sql | 7 +- ...t_stream_name_conflict_stream_name_ab3.sql | 3 +- .../conflict_stream_scalar_ab1.sql | 7 +- .../conflict_stream_scalar_ab2.sql | 7 +- .../conflict_stream_scalar_ab3.sql | 3 +- ..._columns_resulting_into_long_names_ab1.sql | 8 +- ..._columns_resulting_into_long_names_ab2.sql | 8 +- ...ing_into_long_names_partition_DATA_ab1.sql | 9 +- ...ing_into_long_names_partition_DATA_ab2.sql | 7 +- ...ing_into_long_names_partition_DATA_ab3.sql | 3 +- ...esulting_into_long_names_partition_ab1.sql | 9 +- ...esulting_into_long_names_partition_ab2.sql | 7 +- ...esulting_into_long_names_partition_ab3.sql | 3 +- ...es_partition_column___with__quotes_ab1.sql | 9 +- ...es_partition_column___with__quotes_ab2.sql | 7 +- ...es_partition_column___with__quotes_ab3.sql | 3 +- ..._names_partition_double_array_data_ab1.sql | 9 +- ..._names_partition_double_array_data_ab2.sql | 7 +- ..._names_partition_double_array_data_ab3.sql | 3 +- ...amespace_resulting_into_long_names_ab1.sql | 7 +- ...amespace_resulting_into_long_names_ab2.sql | 7 +- ...amespace_resulting_into_long_names_ab3.sql | 3 +- .../test_normalization/unnest_alias_ab1.sql | 8 +- .../test_normalization/unnest_alias_ab2.sql | 8 +- .../test_normalization/unnest_alias_ab3.sql | 4 +- .../unnest_alias_children_ab1.sql | 9 +- .../unnest_alias_children_ab2.sql | 7 +- .../unnest_alias_children_ab3.sql | 3 +- .../unnest_alias_children_owner_ab1.sql | 9 +- .../unnest_alias_children_owner_ab2.sql | 7 +- .../unnest_alias_children_owner_ab3.sql | 3 +- ...amespace_resulting_into_long_names_ab1.sql | 7 +- ...amespace_resulting_into_long_names_ab2.sql | 7 +- ...amespace_resulting_into_long_names_ab3.sql | 3 +- ..._columns_resulting_into_long_names_scd.sql | 86 ++++++++++ ...plex_columns_resulting_into_long_names.sql | 13 +- .../test_normalization/unnest_alias.sql | 10 +- .../conflict_stream_array.sql | 7 +- .../conflict_stream_name.sql | 7 +- ...flict_stream_name_conflict_stream_name.sql | 7 +- ...flict_stream_name_conflict_stream_name.sql | 7 +- .../conflict_stream_scalar.sql | 7 +- ...ns_resulting_into_long_names_partition.sql | 7 +- ...sulting_into_long_names_partition_DATA.sql | 7 +- ..._names_partition_column___with__quotes.sql | 7 +- ...long_names_partition_double_array_data.sql | 7 +- ...ut_namespace_resulting_into_long_names.sql | 7 +- .../unnest_alias_children.sql | 7 +- .../unnest_alias_children_owner.sql | 7 +- ...th_namespace_resulting_into_long_names.sql | 7 +- ..._columns_resulting_into_long_names_ab3.sql | 4 +- .../tmp_test_data_check_row_counts.sql | 19 --- .../conflict_stream_array_ab1.sql | 13 +- .../conflict_stream_array_ab2.sql | 13 +- .../conflict_stream_array_ab3.sql | 9 +- .../conflict_stream_name_ab1.sql | 13 +- .../conflict_stream_name_ab2.sql | 13 +- .../conflict_stream_name_ab3.sql | 9 +- ...t_stream_name_conflict_stream_name_ab1.sql | 15 +- ...t_stream_name_conflict_stream_name_ab2.sql | 13 +- ...t_stream_name_conflict_stream_name_ab3.sql | 9 +- ...t_stream_name_conflict_stream_name_ab1.sql | 15 +- ...t_stream_name_conflict_stream_name_ab2.sql | 13 +- ...t_stream_name_conflict_stream_name_ab3.sql | 9 +- .../conflict_stream_scalar_ab1.sql | 13 +- .../conflict_stream_scalar_ab2.sql | 13 +- .../conflict_stream_scalar_ab3.sql | 9 +- ..._columns_resulting_into_long_names_ab1.sql | 14 +- ..._columns_resulting_into_long_names_ab2.sql | 14 +- ..._columns_resulting_into_long_names_ab4.sql | 11 -- ...ing_into_long_names_partition_DATA_ab1.sql | 17 +- ...ing_into_long_names_partition_DATA_ab2.sql | 13 +- ...ing_into_long_names_partition_DATA_ab3.sql | 9 +- ...esulting_into_long_names_partition_ab1.sql | 15 +- ...esulting_into_long_names_partition_ab2.sql | 13 +- ...esulting_into_long_names_partition_ab3.sql | 9 +- ...es_partition_column___with__quotes_ab1.sql | 17 +- ...es_partition_column___with__quotes_ab2.sql | 13 +- ...es_partition_column___with__quotes_ab3.sql | 9 +- ..._names_partition_double_array_data_ab1.sql | 17 +- ..._names_partition_double_array_data_ab2.sql | 13 +- ..._names_partition_double_array_data_ab3.sql | 9 +- ...amespace_resulting_into_long_names_ab1.sql | 13 +- ...amespace_resulting_into_long_names_ab2.sql | 13 +- ...amespace_resulting_into_long_names_ab3.sql | 9 +- .../test_normalization/unnest_alias_ab1.sql | 14 +- .../test_normalization/unnest_alias_ab2.sql | 14 +- .../test_normalization/unnest_alias_ab3.sql | 10 +- .../unnest_alias_children_ab1.sql | 17 +- .../unnest_alias_children_ab2.sql | 13 +- .../unnest_alias_children_ab3.sql | 9 +- .../unnest_alias_children_owner_ab1.sql | 15 +- .../unnest_alias_children_owner_ab2.sql | 13 +- .../unnest_alias_children_owner_ab3.sql | 9 +- ...amespace_resulting_into_long_names_ab1.sql | 13 +- ...amespace_resulting_into_long_names_ab2.sql | 13 +- ...amespace_resulting_into_long_names_ab3.sql | 9 +- ..._columns_resulting_into_long_names_scd.sql | 109 +++++++++++++ ...plex_columns_resulting_into_long_names.sql | 23 +++ .../test_normalization/unnest_alias.sql | 20 +++ .../conflict_stream_array.sql | 11 +- .../conflict_stream_name.sql | 11 +- ...flict_stream_name_conflict_stream_name.sql | 11 +- ...flict_stream_name_conflict_stream_name.sql | 11 +- .../conflict_stream_scalar.sql | 11 +- ...plex_columns_resulting_into_long_names.sql | 12 -- ...ns_resulting_into_long_names_partition.sql | 11 +- ...sulting_into_long_names_partition_DATA.sql | 11 +- ..._names_partition_column___with__quotes.sql | 11 +- ...long_names_partition_double_array_data.sql | 11 +- ..._columns_resulting_into_long_names_scd.sql | 21 --- ...ut_namespace_resulting_into_long_names.sql | 11 +- .../test_normalization/unnest_alias.sql | 10 -- .../unnest_alias_children.sql | 11 +- .../unnest_alias_children_owner.sql | 11 +- ...th_namespace_resulting_into_long_names.sql | 11 +- ..._columns_resulting_into_long_names_ab3.sql | 10 +- .../conflict_stream_array_ab1.sql | 16 ++ .../conflict_stream_array_ab2.sql | 18 +++ .../conflict_stream_array_ab3.sql | 19 +++ .../conflict_stream_name_ab1.sql | 18 +++ .../conflict_stream_name_ab2.sql | 20 +++ .../conflict_stream_name_ab3.sql | 19 +++ ...t_stream_name_conflict_stream_name_ab1.sql | 19 +++ ...t_stream_name_conflict_stream_name_ab2.sql | 18 +++ ...t_stream_name_conflict_stream_name_ab3.sql | 19 +++ ...t_stream_name_conflict_stream_name_ab1.sql | 17 ++ ...t_stream_name_conflict_stream_name_ab2.sql | 18 +++ ...t_stream_name_conflict_stream_name_ab3.sql | 19 +++ .../conflict_stream_scalar_ab1.sql | 16 ++ .../conflict_stream_scalar_ab2.sql | 20 +++ .../conflict_stream_scalar_ab3.sql | 19 +++ ..._columns_resulting_into_long_names_ab1.sql | 20 +++ ..._columns_resulting_into_long_names_ab2.sql | 24 +++ ...ing_into_long_names_partition_DATA_ab1.sql | 19 +++ ...ing_into_long_names_partition_DATA_ab2.sql | 18 +++ ...ing_into_long_names_partition_DATA_ab3.sql | 19 +++ ...esulting_into_long_names_partition_ab1.sql | 19 +++ ...esulting_into_long_names_partition_ab2.sql | 18 +++ ...esulting_into_long_names_partition_ab3.sql | 23 +++ ...es_partition_column___with__quotes_ab1.sql | 19 +++ ...es_partition_column___with__quotes_ab2.sql | 18 +++ ...es_partition_column___with__quotes_ab3.sql | 19 +++ ..._names_partition_double_array_data_ab1.sql | 19 +++ ..._names_partition_double_array_data_ab2.sql | 18 +++ ..._names_partition_double_array_data_ab3.sql | 19 +++ ...amespace_resulting_into_long_names_ab1.sql | 16 ++ ...amespace_resulting_into_long_names_ab2.sql | 20 +++ ...amespace_resulting_into_long_names_ab3.sql | 19 +++ .../test_normalization/unnest_alias_ab1.sql | 17 ++ .../test_normalization/unnest_alias_ab2.sql | 19 +++ .../test_normalization/unnest_alias_ab3.sql | 20 +++ .../unnest_alias_children_ab1.sql | 22 +++ .../unnest_alias_children_ab2.sql | 21 +++ .../unnest_alias_children_ab3.sql | 21 +++ .../unnest_alias_children_owner_ab1.sql | 17 ++ .../unnest_alias_children_owner_ab2.sql | 18 +++ .../unnest_alias_children_owner_ab3.sql | 19 +++ ...amespace_resulting_into_long_names_ab1.sql | 16 ++ ...amespace_resulting_into_long_names_ab2.sql | 20 +++ ...amespace_resulting_into_long_names_ab3.sql | 19 +++ ..._columns_resulting_into_long_names_scd.sql | 27 ++++ ...plex_columns_resulting_into_long_names.sql | 27 ++++ .../test_normalization/unnest_alias.sql | 27 ++++ .../conflict_stream_array.sql | 21 +++ .../conflict_stream_name.sql | 21 +++ ...flict_stream_name_conflict_stream_name.sql | 21 +++ ...flict_stream_name_conflict_stream_name.sql | 21 +++ .../conflict_stream_scalar.sql | 21 +++ ...ns_resulting_into_long_names_partition.sql | 23 +++ ...sulting_into_long_names_partition_DATA.sql | 21 +++ ..._names_partition_column___with__quotes.sql | 21 +++ ...long_names_partition_double_array_data.sql | 21 +++ ...ut_namespace_resulting_into_long_names.sql | 21 +++ .../unnest_alias_children.sql | 22 +++ .../unnest_alias_children_owner.sql | 21 +++ ...th_namespace_resulting_into_long_names.sql | 21 +++ ..._columns_resulting_into_long_names_ab3.sql | 22 +++ .../dedup_cdc_excluded_ab4.sql | 15 -- .../dedup_exchange_rate_ab4.sql | 15 -- .../test_normalization/pos_dedup_cdcx_ab4.sql | 15 -- .../dedup_cdc_excluded_scd.sql | 32 ---- .../dedup_exchange_rate_scd.sql | 38 ----- .../test_normalization/pos_dedup_cdcx_scd.sql | 32 ---- .../dedup_cdc_excluded_ab1.sql | 9 +- .../dedup_cdc_excluded_ab2.sql | 11 +- .../dedup_exchange_rate_ab1.sql | 8 +- .../dedup_exchange_rate_ab2.sql | 8 +- .../test_normalization/exchange_rate_ab1.sql | 8 +- .../test_normalization/exchange_rate_ab2.sql | 8 +- .../test_normalization/exchange_rate_ab3.sql | 4 +- .../test_normalization/pos_dedup_cdcx_ab1.sql | 7 +- .../test_normalization/pos_dedup_cdcx_ab2.sql | 7 +- .../renamed_dedup_cdc_excluded_ab1.sql | 16 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 18 +++ .../dedup_cdc_excluded_scd.sql | 98 ++++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 82 ++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 14 +- .../dedup_exchange_rate.sql | 13 +- .../test_normalization/exchange_rate.sql | 10 +- .../renamed_dedup_cdc_excluded.sql | 23 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 104 ++++++++++++ .../test_normalization/pos_dedup_cdcx.sql | 10 +- .../dedup_cdc_excluded_ab3.sql | 6 +- .../dedup_exchange_rate_ab3.sql | 4 +- .../test_normalization/pos_dedup_cdcx_ab3.sql | 3 +- .../renamed_dedup_cdc_excluded_ab3.sql | 18 +++ .../tmp_test_data_check_row_counts.sql | 43 ----- .../dedup_cdc_excluded_ab1.sql | 15 +- .../dedup_cdc_excluded_ab2.sql | 15 +- .../dedup_cdc_excluded_ab4.sql | 11 -- .../dedup_exchange_rate_ab1.sql | 14 +- .../dedup_exchange_rate_ab2.sql | 14 +- .../dedup_exchange_rate_ab4.sql | 11 -- .../test_normalization/exchange_rate_ab1.sql | 14 +- .../test_normalization/exchange_rate_ab2.sql | 14 +- .../test_normalization/exchange_rate_ab3.sql | 10 +- .../test_normalization/pos_dedup_cdcx_ab1.sql | 13 +- .../test_normalization/pos_dedup_cdcx_ab2.sql | 13 +- .../test_normalization/pos_dedup_cdcx_ab4.sql | 11 -- .../renamed_dedup_cdc_excluded_ab1.sql | 18 +++ .../renamed_dedup_cdc_excluded_ab2.sql | 18 +++ .../dedup_cdc_excluded_scd.sql | 113 +++++++++++++ .../dedup_exchange_rate_scd.sql | 4 +- .../renamed_dedup_cdc_excluded_scd.sql | 105 +++++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 25 +++ .../dedup_exchange_rate.sql | 28 ++++ .../test_normalization/exchange_rate.sql | 26 +++ .../renamed_dedup_cdc_excluded.sql | 21 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 115 ++++++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 15 -- .../dedup_cdc_excluded_scd.sql | 24 --- .../dedup_exchange_rate.sql | 17 -- .../dedup_exchange_rate_scd.sql | 26 --- .../test_normalization/exchange_rate.sql | 16 -- .../test_normalization/pos_dedup_cdcx.sql | 14 +- .../test_normalization/pos_dedup_cdcx_scd.sql | 24 --- .../dedup_cdc_excluded_ab3.sql | 22 +++ .../dedup_exchange_rate_ab3.sql | 10 +- .../test_normalization/pos_dedup_cdcx_ab3.sql | 9 +- .../renamed_dedup_cdc_excluded_ab3.sql | 18 +++ .../models/generated/sources.yml | 1 + .../dedup_cdc_excluded_ab1.sql | 20 +++ .../dedup_cdc_excluded_ab2.sql | 30 ++++ .../dedup_exchange_rate_ab1.sql | 23 +++ .../dedup_exchange_rate_ab2.sql | 39 +++++ .../test_normalization/exchange_rate_ab1.sql | 23 +++ .../test_normalization/exchange_rate_ab2.sql | 39 +++++ .../test_normalization/exchange_rate_ab3.sql | 32 ++++ .../test_normalization/pos_dedup_cdcx_ab1.sql | 20 +++ .../test_normalization/pos_dedup_cdcx_ab2.sql | 32 ++++ .../renamed_dedup_cdc_excluded_ab1.sql | 16 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 18 +++ .../dedup_cdc_excluded_scd.sql | 27 ++++ .../renamed_dedup_cdc_excluded_scd.sql | 27 ++++ .../test_normalization/dedup_cdc_excluded.sql | 27 ++++ .../dedup_exchange_rate.sql | 27 ++++ .../test_normalization/exchange_rate.sql | 27 ++++ .../renamed_dedup_cdc_excluded.sql | 27 ++++ .../test_normalization/pos_dedup_cdcx_scd.sql | 104 ++++++++++++ .../test_normalization/pos_dedup_cdcx.sql | 27 ++++ .../dedup_cdc_excluded_ab3.sql | 26 +++ .../dedup_exchange_rate_ab3.sql | 32 ++++ .../test_normalization/pos_dedup_cdcx_ab3.sql | 27 ++++ .../renamed_dedup_cdc_excluded_ab3.sql | 18 +++ .../dedup_exchange_rate_ab1.sql | 23 +++ .../dedup_exchange_rate_ab2.sql | 39 +++++ .../test_normalization/exchange_rate_ab1.sql | 23 +++ .../test_normalization/exchange_rate_ab2.sql | 39 +++++ .../test_normalization/exchange_rate_ab3.sql | 32 ++++ .../renamed_dedup_cdc_excluded_ab1.sql | 20 +++ .../renamed_dedup_cdc_excluded_ab2.sql | 30 ++++ .../renamed_dedup_cdc_excluded_scd.sql | 27 ++++ .../dedup_exchange_rate.sql | 27 ++++ .../test_normalization/exchange_rate.sql | 27 ++++ .../renamed_dedup_cdc_excluded.sql | 27 ++++ .../dedup_exchange_rate_ab3.sql | 32 ++++ .../renamed_dedup_cdc_excluded_ab3.sql | 26 +++ ...eam_with_co__lting_into_long_names_ab4.sql | 14 -- ...eam_with_co__lting_into_long_names_scd.sql | 66 -------- .../conflict_stream_array_ab1.sql | 5 +- .../conflict_stream_array_ab2.sql | 5 +- .../conflict_stream_array_ab3.sql | 1 + ...tream_name____conflict_stream_name_ab1.sql | 7 +- ...tream_name____conflict_stream_name_ab2.sql | 5 +- ...tream_name____conflict_stream_name_ab3.sql | 1 + .../conflict_stream_name_ab1.sql | 5 +- .../conflict_stream_name_ab2.sql | 5 +- .../conflict_stream_name_ab3.sql | 1 + ...t_stream_name_conflict_stream_name_ab1.sql | 7 +- ...t_stream_name_conflict_stream_name_ab2.sql | 5 +- ...t_stream_name_conflict_stream_name_ab3.sql | 1 + .../conflict_stream_scalar_ab1.sql | 5 +- .../conflict_stream_scalar_ab2.sql | 5 +- .../conflict_stream_scalar_ab3.sql | 1 + ...eam_with_co___long_names_partition_ab1.sql | 7 +- ...eam_with_co___long_names_partition_ab2.sql | 5 +- ...eam_with_co___long_names_partition_ab3.sql | 1 + ...eam_with_co___names_partition_data_ab1.sql | 9 +- ...eam_with_co___names_partition_data_ab2.sql | 5 +- ...eam_with_co___names_partition_data_ab3.sql | 1 + ...eam_with_co__column___with__quotes_ab1.sql | 9 +- ...eam_with_co__column___with__quotes_ab2.sql | 5 +- ...eam_with_co__column___with__quotes_ab3.sql | 1 + ...eam_with_co__ion_double_array_data_ab1.sql | 9 +- ...eam_with_co__ion_double_array_data_ab2.sql | 5 +- ...eam_with_co__ion_double_array_data_ab3.sql | 1 + ...eam_with_co__lting_into_long_names_ab1.sql | 6 +- ...eam_with_co__lting_into_long_names_ab2.sql | 6 +- ..._stream_wit__lting_into_long_names_ab1.sql | 5 +- ..._stream_wit__lting_into_long_names_ab2.sql | 5 +- ..._stream_wit__lting_into_long_names_ab3.sql | 1 + .../test_normalization/unnest_alias_ab1.sql | 6 +- .../test_normalization/unnest_alias_ab2.sql | 6 +- .../test_normalization/unnest_alias_ab3.sql | 2 + .../unnest_alias_children_ab1.sql | 9 +- .../unnest_alias_children_ab2.sql | 5 +- .../unnest_alias_children_ab3.sql | 1 + .../unnest_alias_children_owner_ab1.sql | 7 +- .../unnest_alias_children_owner_ab2.sql | 5 +- .../unnest_alias_children_owner_ab3.sql | 1 + ...eam_with_na__lting_into_long_names_ab1.sql | 5 +- ...eam_with_na__lting_into_long_names_ab2.sql | 5 +- ...eam_with_na__lting_into_long_names_ab3.sql | 1 + ...eam_with_co__lting_into_long_names_scd.sql | 125 +++++++++++++++ ..._stream_with_co__lting_into_long_names.sql | 38 +++-- .../test_normalization/unnest_alias.sql | 35 +++-- .../conflict_stream_array.sql | 3 + .../conflict_stream_name.sql | 3 + ...ct_stream_name____conflict_stream_name.sql | 3 + ...flict_stream_name_conflict_stream_name.sql | 3 + .../conflict_stream_scalar.sql | 3 + ..._stream_with_co___long_names_partition.sql | 3 + ..._stream_with_co___names_partition_data.sql | 3 + ..._stream_with_co__column___with__quotes.sql | 3 + ..._stream_with_co__ion_double_array_data.sql | 3 + ...sted_stream_wit__lting_into_long_names.sql | 3 + .../unnest_alias_children.sql | 3 + .../unnest_alias_children_owner.sql | 3 + ..._stream_with_na__lting_into_long_names.sql | 3 + ...eam_with_co__lting_into_long_names_ab3.sql | 2 + .../tmp_test_data_check_row_counts.sql | 19 --- .../conflict_stream_array_ab1.sql | 11 +- .../conflict_stream_array_ab2.sql | 11 +- .../conflict_stream_array_ab3.sql | 7 +- ...tream_name____conflict_stream_name_ab1.sql | 13 +- ...tream_name____conflict_stream_name_ab2.sql | 11 +- ...tream_name____conflict_stream_name_ab3.sql | 7 +- .../conflict_stream_name_ab1.sql | 11 +- .../conflict_stream_name_ab2.sql | 11 +- .../conflict_stream_name_ab3.sql | 7 +- ...t_stream_name_conflict_stream_name_ab1.sql | 13 +- ...t_stream_name_conflict_stream_name_ab2.sql | 11 +- ...t_stream_name_conflict_stream_name_ab3.sql | 7 +- .../conflict_stream_scalar_ab1.sql | 11 +- .../conflict_stream_scalar_ab2.sql | 11 +- .../conflict_stream_scalar_ab3.sql | 7 +- ...eam_with_co___long_names_partition_ab1.sql | 13 +- ...eam_with_co___long_names_partition_ab2.sql | 11 +- ...eam_with_co___long_names_partition_ab3.sql | 7 +- ...eam_with_co___names_partition_data_ab1.sql | 15 +- ...eam_with_co___names_partition_data_ab2.sql | 11 +- ...eam_with_co___names_partition_data_ab3.sql | 7 +- ...eam_with_co__column___with__quotes_ab1.sql | 15 +- ...eam_with_co__column___with__quotes_ab2.sql | 11 +- ...eam_with_co__column___with__quotes_ab3.sql | 7 +- ...eam_with_co__ion_double_array_data_ab1.sql | 15 +- ...eam_with_co__ion_double_array_data_ab2.sql | 11 +- ...eam_with_co__ion_double_array_data_ab3.sql | 7 +- ...eam_with_co__lting_into_long_names_ab1.sql | 12 +- ...eam_with_co__lting_into_long_names_ab2.sql | 12 +- ...eam_with_co__lting_into_long_names_ab4.sql | 11 -- ..._stream_wit__lting_into_long_names_ab1.sql | 11 +- ..._stream_wit__lting_into_long_names_ab2.sql | 11 +- ..._stream_wit__lting_into_long_names_ab3.sql | 7 +- .../test_normalization/unnest_alias_ab1.sql | 12 +- .../test_normalization/unnest_alias_ab2.sql | 12 +- .../test_normalization/unnest_alias_ab3.sql | 8 +- .../unnest_alias_children_ab1.sql | 15 +- .../unnest_alias_children_ab2.sql | 11 +- .../unnest_alias_children_ab3.sql | 7 +- .../unnest_alias_children_owner_ab1.sql | 13 +- .../unnest_alias_children_owner_ab2.sql | 11 +- .../unnest_alias_children_owner_ab3.sql | 7 +- ...eam_with_na__lting_into_long_names_ab1.sql | 11 +- ...eam_with_na__lting_into_long_names_ab2.sql | 11 +- ...eam_with_na__lting_into_long_names_ab3.sql | 7 +- ...eam_with_co__lting_into_long_names_scd.sql | 107 +++++++++++++ ..._stream_with_co__lting_into_long_names.sql | 13 +- .../test_normalization/unnest_alias.sql | 18 +++ .../conflict_stream_array.sql | 9 +- .../conflict_stream_name.sql | 9 +- ...ct_stream_name____conflict_stream_name.sql | 9 +- ...flict_stream_name_conflict_stream_name.sql | 9 +- .../conflict_stream_scalar.sql | 9 +- ..._stream_with_co___long_names_partition.sql | 9 +- ..._stream_with_co___names_partition_data.sql | 9 +- ..._stream_with_co__column___with__quotes.sql | 9 +- ..._stream_with_co__ion_double_array_data.sql | 9 +- ...eam_with_co__lting_into_long_names_scd.sql | 21 --- ...sted_stream_wit__lting_into_long_names.sql | 9 +- .../test_normalization/unnest_alias.sql | 10 -- .../unnest_alias_children.sql | 9 +- .../unnest_alias_children_owner.sql | 9 +- ..._stream_with_na__lting_into_long_names.sql | 9 +- ...eam_with_co__lting_into_long_names_ab3.sql | 8 +- .../conflict_stream_array_ab1.sql | 15 ++ .../conflict_stream_array_ab2.sql | 16 ++ .../conflict_stream_array_ab3.sql | 20 +++ ...tream_name____conflict_stream_name_ab1.sql | 16 ++ ...tream_name____conflict_stream_name_ab2.sql | 16 ++ ...tream_name____conflict_stream_name_ab3.sql | 19 +++ .../conflict_stream_name_ab1.sql | 15 ++ .../conflict_stream_name_ab2.sql | 16 ++ .../conflict_stream_name_ab3.sql | 19 +++ ...t_stream_name_conflict_stream_name_ab1.sql | 16 ++ ...t_stream_name_conflict_stream_name_ab2.sql | 15 ++ ...t_stream_name_conflict_stream_name_ab3.sql | 19 +++ .../conflict_stream_scalar_ab1.sql | 15 ++ .../conflict_stream_scalar_ab2.sql | 18 +++ .../conflict_stream_scalar_ab3.sql | 19 +++ ...eam_with_co___long_names_partition_ab1.sql | 18 +++ ...eam_with_co___long_names_partition_ab2.sql | 17 ++ ...eam_with_co___long_names_partition_ab3.sql | 24 +++ ...eam_with_co___names_partition_data_ab1.sql | 26 +++ ...eam_with_co___names_partition_data_ab2.sql | 16 ++ ...eam_with_co___names_partition_data_ab3.sql | 19 +++ ...eam_with_co__column___with__quotes_ab1.sql | 26 +++ ...eam_with_co__column___with__quotes_ab2.sql | 16 ++ ...eam_with_co__column___with__quotes_ab3.sql | 19 +++ ...eam_with_co__ion_double_array_data_ab1.sql | 26 +++ ...eam_with_co__ion_double_array_data_ab2.sql | 16 ++ ...eam_with_co__ion_double_array_data_ab3.sql | 19 +++ ...eam_with_co__lting_into_long_names_ab1.sql | 17 ++ ...eam_with_co__lting_into_long_names_ab2.sql | 19 +++ ..._stream_wit__lting_into_long_names_ab1.sql | 15 ++ ..._stream_wit__lting_into_long_names_ab2.sql | 17 ++ ..._stream_wit__lting_into_long_names_ab3.sql | 19 +++ .../test_normalization/unnest_alias_ab1.sql | 16 ++ .../test_normalization/unnest_alias_ab2.sql | 18 +++ .../test_normalization/unnest_alias_ab3.sql | 21 +++ .../unnest_alias_children_ab1.sql | 28 ++++ .../unnest_alias_children_ab2.sql | 18 +++ .../unnest_alias_children_ab3.sql | 20 +++ .../unnest_alias_children_owner_ab1.sql | 16 ++ .../unnest_alias_children_owner_ab2.sql | 17 ++ .../unnest_alias_children_owner_ab3.sql | 19 +++ ...eam_with_na__lting_into_long_names_ab1.sql | 15 ++ ...eam_with_na__lting_into_long_names_ab2.sql | 17 ++ ...eam_with_na__lting_into_long_names_ab3.sql | 19 +++ ...eam_with_co__lting_into_long_names_scd.sql | 14 ++ ..._stream_with_co__lting_into_long_names.sql | 14 ++ .../test_normalization/unnest_alias.sql | 14 ++ .../conflict_stream_array.sql | 58 +++++++ .../conflict_stream_name.sql | 58 +++++++ ...ct_stream_name____conflict_stream_name.sql | 58 +++++++ ...flict_stream_name_conflict_stream_name.sql | 58 +++++++ .../conflict_stream_scalar.sql | 58 +++++++ ..._stream_with_co___long_names_partition.sql | 60 +++++++ ..._stream_with_co___names_partition_data.sql | 58 +++++++ ..._stream_with_co__column___with__quotes.sql | 58 +++++++ ..._stream_with_co__ion_double_array_data.sql | 58 +++++++ ...sted_stream_wit__lting_into_long_names.sql | 58 +++++++ .../unnest_alias_children.sql | 59 +++++++ .../unnest_alias_children_owner.sql | 58 +++++++ ..._stream_with_na__lting_into_long_names.sql | 58 +++++++ ...eam_with_co__lting_into_long_names_ab3.sql | 21 +++ .../dedup_cdc_excluded_ab4.sql | 14 -- .../dedup_exchange_rate_ab4.sql | 14 -- .../test_normalization/pos_dedup_cdcx_ab4.sql | 14 -- .../dedup_cdc_excluded_scd.sql | 69 -------- .../dedup_exchange_rate_scd.sql | 73 --------- .../test_normalization/pos_dedup_cdcx_scd.sql | 69 -------- .../dedup_cdc_excluded_ab1.sql | 7 +- .../dedup_cdc_excluded_ab2.sql | 8 +- .../dedup_exchange_rate_ab1.sql | 6 +- .../dedup_exchange_rate_ab2.sql | 6 +- .../test_normalization/exchange_rate_ab1.sql | 6 +- .../test_normalization/exchange_rate_ab2.sql | 6 +- .../test_normalization/exchange_rate_ab3.sql | 2 + .../test_normalization/pos_dedup_cdcx_ab1.sql | 5 +- .../test_normalization/pos_dedup_cdcx_ab2.sql | 5 +- .../renamed_dedup_cdc_excluded_ab1.sql | 15 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 17 ++ .../dedup_cdc_excluded_scd.sql | 133 ++++++++++++++++ .../dedup_exchange_rate_scd.sql | 139 ++++++++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 121 ++++++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 39 +++-- .../dedup_exchange_rate.sql | 38 +++-- .../test_normalization/exchange_rate.sql | 35 +++-- .../renamed_dedup_cdc_excluded.sql | 63 ++++++++ .../test_normalization/pos_dedup_cdcx_scd.sql | 134 ++++++++++++++++ .../test_normalization/pos_dedup_cdcx.sql | 6 +- .../dedup_cdc_excluded_ab3.sql | 3 +- .../dedup_exchange_rate_ab3.sql | 2 + .../test_normalization/pos_dedup_cdcx_ab3.sql | 1 + .../renamed_dedup_cdc_excluded_ab3.sql | 19 +++ .../tmp_test_data_check_row_counts.sql | 43 ----- .../dedup_cdc_excluded_ab1.sql | 13 +- .../dedup_cdc_excluded_ab2.sql | 13 +- .../dedup_cdc_excluded_ab4.sql | 11 -- .../dedup_exchange_rate_ab1.sql | 12 +- .../dedup_exchange_rate_ab2.sql | 12 +- .../dedup_exchange_rate_ab4.sql | 11 -- .../test_normalization/exchange_rate_ab1.sql | 12 +- .../test_normalization/exchange_rate_ab2.sql | 12 +- .../test_normalization/exchange_rate_ab3.sql | 8 +- .../test_normalization/pos_dedup_cdcx_ab1.sql | 11 +- .../test_normalization/pos_dedup_cdcx_ab2.sql | 11 +- .../test_normalization/pos_dedup_cdcx_ab4.sql | 11 -- .../renamed_dedup_cdc_excluded_ab1.sql | 16 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 16 ++ .../dedup_cdc_excluded_scd.sql | 111 +++++++++++++ .../dedup_exchange_rate_scd.sql | 121 ++++++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 103 ++++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 14 +- .../dedup_exchange_rate.sql | 13 +- .../test_normalization/exchange_rate.sql | 10 +- .../renamed_dedup_cdc_excluded.sql | 19 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 113 +++++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 15 -- .../dedup_cdc_excluded_scd.sql | 24 --- .../dedup_exchange_rate_scd.sql | 26 --- .../test_normalization/pos_dedup_cdcx.sql | 12 +- .../test_normalization/pos_dedup_cdcx_scd.sql | 24 --- .../dedup_cdc_excluded_ab3.sql | 9 +- .../dedup_exchange_rate_ab3.sql | 8 +- .../test_normalization/pos_dedup_cdcx_ab3.sql | 7 +- .../renamed_dedup_cdc_excluded_ab3.sql | 16 ++ .../models/generated/sources.yml | 1 + .../dedup_cdc_excluded_ab1.sql | 19 +++ .../dedup_cdc_excluded_ab2.sql | 28 ++++ .../dedup_exchange_rate_ab1.sql | 22 +++ .../dedup_exchange_rate_ab2.sql | 32 ++++ .../test_normalization/exchange_rate_ab1.sql | 22 +++ .../test_normalization/exchange_rate_ab2.sql | 32 ++++ .../test_normalization/exchange_rate_ab3.sql | 26 +++ .../test_normalization/pos_dedup_cdcx_ab1.sql | 19 +++ .../test_normalization/pos_dedup_cdcx_ab2.sql | 30 ++++ .../renamed_dedup_cdc_excluded_ab1.sql | 15 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 17 ++ .../dedup_cdc_excluded_scd.sql | 14 ++ .../dedup_exchange_rate_scd.sql | 14 ++ .../renamed_dedup_cdc_excluded_scd.sql | 14 ++ .../test_normalization/dedup_cdc_excluded.sql | 14 ++ .../dedup_exchange_rate.sql | 14 ++ .../test_normalization/exchange_rate.sql | 14 ++ .../renamed_dedup_cdc_excluded.sql | 14 ++ .../test_normalization/pos_dedup_cdcx_scd.sql | 134 ++++++++++++++++ .../test_normalization/pos_dedup_cdcx.sql | 64 ++++++++ .../dedup_cdc_excluded_ab3.sql | 23 +++ .../dedup_exchange_rate_ab3.sql | 26 +++ .../test_normalization/pos_dedup_cdcx_ab3.sql | 23 +++ .../renamed_dedup_cdc_excluded_ab3.sql | 19 +++ ..._stream_with_co_1g_into_long_names_ab4.sql | 13 -- ..._stream_with_co_1g_into_long_names_scd.sql | 26 --- .../conflict_stream_array_ab1.sql | 7 +- .../conflict_stream_array_ab2.sql | 7 +- .../conflict_stream_array_ab3.sql | 1 + ...ct_stream_name__2flict_stream_name_ab1.sql | 9 +- ...ct_stream_name__2flict_stream_name_ab2.sql | 7 +- ...ct_stream_name__2flict_stream_name_ab3.sql | 1 + ...ct_stream_name__3flict_stream_name_ab1.sql | 9 +- ...ct_stream_name__3flict_stream_name_ab2.sql | 7 +- ...ct_stream_name__3flict_stream_name_ab3.sql | 1 + .../conflict_stream_name_ab1.sql | 7 +- .../conflict_stream_name_ab2.sql | 7 +- .../conflict_stream_name_ab3.sql | 1 + .../conflict_stream_scalar_ab1.sql | 7 +- .../conflict_stream_scalar_ab2.sql | 7 +- .../conflict_stream_scalar_ab3.sql | 1 + ..._stream_with_co_1g_into_long_names_ab1.sql | 8 +- ..._stream_with_co_1g_into_long_names_ab2.sql | 8 +- ..._stream_with_co_2g_names_partition_ab1.sql | 9 +- ..._stream_with_co_2g_names_partition_ab2.sql | 7 +- ..._stream_with_co_2g_names_partition_ab3.sql | 1 + ..._stream_with_co_3double_array_data_ab1.sql | 11 +- ..._stream_with_co_3double_array_data_ab2.sql | 7 +- ..._stream_with_co_3double_array_data_ab3.sql | 1 + ..._stream_with_co_3es_partition_data_ab1.sql | 11 +- ..._stream_with_co_3es_partition_data_ab2.sql | 7 +- ..._stream_with_co_3es_partition_data_ab3.sql | 1 + ..._stream_with_co_3mn___with__quotes_ab1.sql | 11 +- ..._stream_with_co_3mn___with__quotes_ab2.sql | 7 +- ..._stream_with_co_3mn___with__quotes_ab3.sql | 1 + ...sted_stream_wit_1g_into_long_names_ab1.sql | 7 +- ...sted_stream_wit_1g_into_long_names_ab2.sql | 7 +- ...sted_stream_wit_1g_into_long_names_ab3.sql | 1 + .../test_normalization/unnest_alias_ab1.sql | 8 +- .../test_normalization/unnest_alias_ab2.sql | 8 +- .../test_normalization/unnest_alias_ab3.sql | 2 + .../unnest_alias_children_ab1.sql | 11 +- .../unnest_alias_children_ab2.sql | 7 +- .../unnest_alias_children_ab3.sql | 1 + .../unnest_alias_children_owner_ab1.sql | 9 +- .../unnest_alias_children_owner_ab2.sql | 7 +- .../unnest_alias_children_owner_ab3.sql | 1 + ..._stream_with_na_1g_into_long_names_ab1.sql | 7 +- ..._stream_with_na_1g_into_long_names_ab2.sql | 7 +- ..._stream_with_na_1g_into_long_names_ab3.sql | 1 + ..._stream_with_co_1g_into_long_names_scd.sql | 70 +++++++++ ..._stream_with_co__lting_into_long_names.sql | 9 +- .../test_normalization/unnest_alias.sql | 6 + .../conflict_stream_array.sql | 5 + .../conflict_stream_name.sql | 5 + ...ct_stream_name____conflict_stream_name.sql | 5 + ...flict_stream_name_conflict_stream_name.sql | 5 + .../conflict_stream_scalar.sql | 5 + ..._stream_with_co___long_names_partition.sql | 5 + ..._stream_with_co___names_partition_data.sql | 5 + ..._stream_with_co__column___with__quotes.sql | 5 + ..._stream_with_co__ion_double_array_data.sql | 5 + ...sted_stream_wit__lting_into_long_names.sql | 5 + .../unnest_alias_children.sql | 5 + .../unnest_alias_children_owner.sql | 5 + ..._stream_with_na__lting_into_long_names.sql | 5 + ..._stream_with_co_1g_into_long_names_ab3.sql | 2 + .../tmp_test_data_check_row_counts.sql | 19 --- .../conflict_stream_array_ab1.sql | 11 +- .../conflict_stream_array_ab2.sql | 11 +- .../conflict_stream_array_ab3.sql | 7 +- ...ct_stream_name__2flict_stream_name_ab1.sql | 13 +- ...ct_stream_name__2flict_stream_name_ab2.sql | 11 +- ...ct_stream_name__2flict_stream_name_ab3.sql | 7 +- ...ct_stream_name__3flict_stream_name_ab1.sql | 13 +- ...ct_stream_name__3flict_stream_name_ab2.sql | 11 +- ...ct_stream_name__3flict_stream_name_ab3.sql | 7 +- .../conflict_stream_name_ab1.sql | 11 +- .../conflict_stream_name_ab2.sql | 11 +- .../conflict_stream_name_ab3.sql | 7 +- .../conflict_stream_scalar_ab1.sql | 11 +- .../conflict_stream_scalar_ab2.sql | 11 +- .../conflict_stream_scalar_ab3.sql | 7 +- ..._stream_with_co_1g_into_long_names_ab1.sql | 12 +- ..._stream_with_co_1g_into_long_names_ab2.sql | 12 +- ..._stream_with_co_1g_into_long_names_ab4.sql | 11 -- ..._stream_with_co_2g_names_partition_ab1.sql | 13 +- ..._stream_with_co_2g_names_partition_ab2.sql | 11 +- ..._stream_with_co_2g_names_partition_ab3.sql | 7 +- ..._stream_with_co_3double_array_data_ab1.sql | 15 +- ..._stream_with_co_3double_array_data_ab2.sql | 11 +- ..._stream_with_co_3double_array_data_ab3.sql | 7 +- ..._stream_with_co_3es_partition_data_ab1.sql | 15 +- ..._stream_with_co_3es_partition_data_ab2.sql | 11 +- ..._stream_with_co_3es_partition_data_ab3.sql | 7 +- ..._stream_with_co_3mn___with__quotes_ab1.sql | 15 +- ..._stream_with_co_3mn___with__quotes_ab2.sql | 11 +- ..._stream_with_co_3mn___with__quotes_ab3.sql | 7 +- ...sted_stream_wit_1g_into_long_names_ab1.sql | 11 +- ...sted_stream_wit_1g_into_long_names_ab2.sql | 11 +- ...sted_stream_wit_1g_into_long_names_ab3.sql | 7 +- .../test_normalization/unnest_alias_ab1.sql | 12 +- .../test_normalization/unnest_alias_ab2.sql | 12 +- .../test_normalization/unnest_alias_ab3.sql | 8 +- .../unnest_alias_children_ab1.sql | 15 +- .../unnest_alias_children_ab2.sql | 11 +- .../unnest_alias_children_ab3.sql | 7 +- .../unnest_alias_children_owner_ab1.sql | 13 +- .../unnest_alias_children_owner_ab2.sql | 11 +- .../unnest_alias_children_owner_ab3.sql | 7 +- ..._stream_with_na_1g_into_long_names_ab1.sql | 11 +- ..._stream_with_na_1g_into_long_names_ab2.sql | 11 +- ..._stream_with_na_1g_into_long_names_ab3.sql | 7 +- ..._stream_with_co_1g_into_long_names_scd.sql | 107 +++++++++++++ ..._stream_with_co__lting_into_long_names.sql | 13 +- .../test_normalization/unnest_alias.sql | 18 +++ .../conflict_stream_array.sql | 9 +- .../conflict_stream_name.sql | 9 +- ...ct_stream_name____conflict_stream_name.sql | 9 +- ...flict_stream_name_conflict_stream_name.sql | 9 +- .../conflict_stream_scalar.sql | 9 +- ..._stream_with_co_1g_into_long_names_scd.sql | 21 --- ..._stream_with_co___long_names_partition.sql | 9 +- ..._stream_with_co___names_partition_data.sql | 9 +- ..._stream_with_co__column___with__quotes.sql | 9 +- ..._stream_with_co__ion_double_array_data.sql | 9 +- ...sted_stream_wit__lting_into_long_names.sql | 9 +- .../test_normalization/unnest_alias.sql | 10 -- .../unnest_alias_children.sql | 9 +- .../unnest_alias_children_owner.sql | 9 +- ..._stream_with_na__lting_into_long_names.sql | 9 +- ..._stream_with_co_1g_into_long_names_ab3.sql | 8 +- .../conflict_stream_array_ab1.sql | 18 +++ .../conflict_stream_array_ab2.sql | 16 ++ .../conflict_stream_array_ab3.sql | 11 ++ ...ct_stream_name__2flict_stream_name_ab1.sql | 20 +++ ...ct_stream_name__2flict_stream_name_ab2.sql | 16 ++ ...ct_stream_name__2flict_stream_name_ab3.sql | 11 ++ ...ct_stream_name__3flict_stream_name_ab1.sql | 18 +++ ...ct_stream_name__3flict_stream_name_ab2.sql | 16 ++ ...ct_stream_name__3flict_stream_name_ab3.sql | 11 ++ .../conflict_stream_name_ab1.sql | 20 +++ .../conflict_stream_name_ab2.sql | 16 ++ .../conflict_stream_name_ab3.sql | 11 ++ .../conflict_stream_scalar_ab1.sql | 18 +++ .../conflict_stream_scalar_ab2.sql | 18 +++ .../conflict_stream_scalar_ab3.sql | 11 ++ ..._stream_with_co_1g_into_long_names_ab1.sql | 23 +++ ..._stream_with_co_1g_into_long_names_ab2.sql | 18 +++ ..._stream_with_co_2g_names_partition_ab1.sql | 22 +++ ..._stream_with_co_2g_names_partition_ab2.sql | 18 +++ ..._stream_with_co_2g_names_partition_ab3.sql | 11 ++ ..._stream_with_co_3double_array_data_ab1.sql | 64 ++++++++ ..._stream_with_co_3double_array_data_ab2.sql | 16 ++ ..._stream_with_co_3double_array_data_ab3.sql | 11 ++ ..._stream_with_co_3es_partition_data_ab1.sql | 64 ++++++++ ..._stream_with_co_3es_partition_data_ab2.sql | 16 ++ ..._stream_with_co_3es_partition_data_ab3.sql | 11 ++ ..._stream_with_co_3mn___with__quotes_ab1.sql | 64 ++++++++ ..._stream_with_co_3mn___with__quotes_ab2.sql | 16 ++ ..._stream_with_co_3mn___with__quotes_ab3.sql | 11 ++ ...sted_stream_wit_1g_into_long_names_ab1.sql | 18 +++ ...sted_stream_wit_1g_into_long_names_ab2.sql | 16 ++ ...sted_stream_wit_1g_into_long_names_ab3.sql | 11 ++ .../test_normalization/unnest_alias_ab1.sql | 19 +++ .../test_normalization/unnest_alias_ab2.sql | 19 +++ .../test_normalization/unnest_alias_ab3.sql | 12 ++ .../unnest_alias_children_ab1.sql | 68 ++++++++ .../unnest_alias_children_ab2.sql | 19 +++ .../unnest_alias_children_ab3.sql | 11 ++ .../unnest_alias_children_owner_ab1.sql | 18 +++ .../unnest_alias_children_owner_ab2.sql | 18 +++ .../unnest_alias_children_owner_ab3.sql | 11 ++ ..._stream_with_na_1g_into_long_names_ab1.sql | 18 +++ ..._stream_with_na_1g_into_long_names_ab2.sql | 16 ++ ..._stream_with_na_1g_into_long_names_ab3.sql | 11 ++ ..._stream_with_co_1g_into_long_names_scd.sql | 70 +++++++++ ..._stream_with_co__lting_into_long_names.sql | 24 +++ .../test_normalization/unnest_alias.sql | 21 +++ .../conflict_stream_array.sql | 20 +++ .../conflict_stream_name.sql | 20 +++ ...ct_stream_name____conflict_stream_name.sql | 20 +++ ...flict_stream_name_conflict_stream_name.sql | 20 +++ .../conflict_stream_scalar.sql | 20 +++ ..._stream_with_co___long_names_partition.sql | 22 +++ ..._stream_with_co___names_partition_data.sql | 20 +++ ..._stream_with_co__column___with__quotes.sql | 20 +++ ..._stream_with_co__ion_double_array_data.sql | 20 +++ ...sted_stream_wit__lting_into_long_names.sql | 20 +++ .../unnest_alias_children.sql | 21 +++ .../unnest_alias_children_owner.sql | 20 +++ ..._stream_with_na__lting_into_long_names.sql | 20 +++ ..._stream_with_co_1g_into_long_names_ab3.sql | 12 ++ .../dedup_cdc_excluded_ab4.sql | 13 -- .../dedup_exchange_rate_ab4.sql | 13 -- .../test_normalization/pos_dedup_cdcx_ab4.sql | 13 -- .../dedup_cdc_excluded_scd.sql | 29 ---- .../dedup_exchange_rate_scd.sql | 31 ---- .../test_normalization/pos_dedup_cdcx_scd.sql | 29 ---- .../dedup_cdc_excluded_ab1.sql | 10 +- .../dedup_cdc_excluded_ab2.sql | 9 +- .../dedup_exchange_rate_ab1.sql | 8 +- .../dedup_exchange_rate_ab2.sql | 8 +- .../test_normalization/exchange_rate_ab1.sql | 8 +- .../test_normalization/exchange_rate_ab2.sql | 8 +- .../test_normalization/exchange_rate_ab3.sql | 2 + .../test_normalization/pos_dedup_cdcx_ab1.sql | 7 +- .../test_normalization/pos_dedup_cdcx_ab2.sql | 7 +- .../renamed_dedup_cdc_excluded_ab1.sql | 17 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 18 +++ .../dedup_cdc_excluded_scd.sql | 74 +++++++++ .../dedup_exchange_rate_scd.sql | 80 ++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 66 ++++++++ .../test_normalization/dedup_cdc_excluded.sql | 10 +- .../dedup_exchange_rate.sql | 9 +- .../test_normalization/exchange_rate.sql | 6 + .../renamed_dedup_cdc_excluded.sql | 22 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 76 +++++++++ .../test_normalization/pos_dedup_cdcx.sql | 8 +- .../dedup_cdc_excluded_ab3.sql | 4 +- .../dedup_exchange_rate_ab3.sql | 2 + .../test_normalization/pos_dedup_cdcx_ab3.sql | 1 + .../renamed_dedup_cdc_excluded_ab3.sql | 12 ++ .../tmp_test_data_check_row_counts.sql | 43 ----- .../dedup_cdc_excluded_ab1.sql | 13 +- .../dedup_cdc_excluded_ab2.sql | 13 +- .../dedup_cdc_excluded_ab4.sql | 11 -- .../dedup_exchange_rate_ab1.sql | 12 +- .../dedup_exchange_rate_ab2.sql | 12 +- .../dedup_exchange_rate_ab4.sql | 11 -- .../test_normalization/exchange_rate_ab1.sql | 12 +- .../test_normalization/exchange_rate_ab2.sql | 12 +- .../test_normalization/exchange_rate_ab3.sql | 8 +- .../test_normalization/pos_dedup_cdcx_ab1.sql | 11 +- .../test_normalization/pos_dedup_cdcx_ab2.sql | 11 +- .../test_normalization/pos_dedup_cdcx_ab4.sql | 11 -- .../renamed_dedup_cdc_excluded_ab1.sql | 16 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 16 ++ .../dedup_cdc_excluded_scd.sql | 111 +++++++++++++ .../dedup_exchange_rate_scd.sql | 121 ++++++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 103 ++++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 14 +- .../dedup_exchange_rate.sql | 13 +- .../test_normalization/exchange_rate.sql | 10 +- .../renamed_dedup_cdc_excluded.sql | 19 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 113 +++++++++++++ .../dedup_cdc_excluded_scd.sql | 24 --- .../dedup_exchange_rate_scd.sql | 26 --- .../test_normalization/pos_dedup_cdcx.sql | 12 +- .../test_normalization/pos_dedup_cdcx_scd.sql | 24 --- .../dedup_cdc_excluded_ab3.sql | 9 +- .../dedup_exchange_rate_ab3.sql | 8 +- .../test_normalization/pos_dedup_cdcx_ab3.sql | 7 +- .../renamed_dedup_cdc_excluded_ab3.sql | 16 ++ .../models/generated/sources.yml | 1 + .../dedup_cdc_excluded_ab1.sql | 25 +++ .../dedup_cdc_excluded_ab2.sql | 28 ++++ .../dedup_exchange_rate_ab1.sql | 31 ++++ .../dedup_exchange_rate_ab2.sql | 34 ++++ .../test_normalization/exchange_rate_ab1.sql | 31 ++++ .../test_normalization/exchange_rate_ab2.sql | 34 ++++ .../test_normalization/exchange_rate_ab3.sql | 12 ++ .../test_normalization/pos_dedup_cdcx_ab1.sql | 26 +++ .../test_normalization/pos_dedup_cdcx_ab2.sql | 30 ++++ .../renamed_dedup_cdc_excluded_ab1.sql | 17 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 18 +++ .../dedup_cdc_excluded_scd.sql | 74 +++++++++ .../dedup_exchange_rate_scd.sql | 80 ++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 66 ++++++++ .../test_normalization/dedup_cdc_excluded.sql | 26 +++ .../dedup_exchange_rate.sql | 29 ++++ .../test_normalization/exchange_rate.sql | 27 ++++ .../renamed_dedup_cdc_excluded.sql | 22 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 76 +++++++++ .../test_normalization/pos_dedup_cdcx.sql | 26 +++ .../dedup_cdc_excluded_ab3.sql | 12 ++ .../dedup_exchange_rate_ab3.sql | 12 ++ .../test_normalization/pos_dedup_cdcx_ab3.sql | 11 ++ .../renamed_dedup_cdc_excluded_ab3.sql | 12 ++ .../dedup_cdc_excluded_scd.sql | 100 ------------ .../dedup_cdc_excluded_scd.sql | 93 +++++++++++ .../dedup_exchange_rate_scd.sql | 99 ++++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 79 ++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 9 +- .../dedup_exchange_rate.sql | 8 +- .../test_normalization/exchange_rate.sql | 25 ++- .../renamed_dedup_cdc_excluded.sql | 20 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 98 ++++++++++++ .../test_normalization/pos_dedup_cdcx.sql | 8 +- .../dedup_cdc_excluded_ab3.sql | 72 +++++++++ .../dedup_exchange_rate_ab3.sql} | 59 ++----- .../pos_dedup_cdcx_ab3.sql} | 56 ++----- .../renamed_dedup_cdc_excluded_ab3.sql | 46 ++++++ .../tmp_test_data_check_row_counts.sql | 43 ----- .../dedup_cdc_excluded_ab1.sql | 13 +- .../dedup_cdc_excluded_ab2.sql | 13 +- .../dedup_cdc_excluded_ab4.sql | 11 -- .../dedup_exchange_rate_ab1.sql | 12 +- .../dedup_exchange_rate_ab2.sql | 12 +- .../dedup_exchange_rate_ab4.sql | 11 -- .../test_normalization/exchange_rate_ab1.sql | 12 +- .../test_normalization/exchange_rate_ab2.sql | 12 +- .../test_normalization/exchange_rate_ab3.sql | 8 +- .../test_normalization/pos_dedup_cdcx_ab1.sql | 11 +- .../test_normalization/pos_dedup_cdcx_ab2.sql | 11 +- .../test_normalization/pos_dedup_cdcx_ab4.sql | 11 -- .../renamed_dedup_cdc_excluded_ab1.sql | 16 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 16 ++ .../dedup_cdc_excluded_scd.sql | 111 +++++++++++++ .../dedup_exchange_rate_scd.sql | 121 ++++++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 103 ++++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 14 +- .../dedup_exchange_rate.sql | 13 +- .../test_normalization/exchange_rate.sql | 10 +- .../renamed_dedup_cdc_excluded.sql | 19 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 113 +++++++++++++ .../dedup_cdc_excluded_scd.sql | 24 --- .../dedup_exchange_rate_scd.sql | 26 --- .../test_normalization/pos_dedup_cdcx.sql | 12 +- .../test_normalization/pos_dedup_cdcx_scd.sql | 24 --- .../dedup_cdc_excluded_ab3.sql | 9 +- .../dedup_exchange_rate_ab3.sql | 8 +- .../test_normalization/pos_dedup_cdcx_ab3.sql | 7 +- .../renamed_dedup_cdc_excluded_ab3.sql | 16 ++ .../models/generated/sources.yml | 1 + .../dedup_cdc_excluded_scd.sql | 93 +++++++++++ .../dedup_exchange_rate_scd.sql | 99 ++++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 79 ++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 24 +++ .../dedup_exchange_rate.sql | 27 ++++ .../test_normalization/exchange_rate.sql | 114 ++++++++++++++ .../renamed_dedup_cdc_excluded.sql | 20 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 98 ++++++++++++ .../test_normalization/pos_dedup_cdcx.sql | 25 +++ .../dedup_cdc_excluded_ab3.sql | 72 +++++++++ .../dedup_exchange_rate_ab3.sql | 91 +++++++++++ .../test_normalization/pos_dedup_cdcx_ab3.sql | 76 +++++++++ .../renamed_dedup_cdc_excluded_ab3.sql | 46 ++++++ ...ream_with_c__lting_into_long_names_ab4.sql | 13 -- ...ream_with_c__lting_into_long_names_scd.sql | 25 --- .../conflict_stream_array_ab1.sql | 5 +- .../conflict_stream_array_ab2.sql | 5 +- .../conflict_stream_array_ab3.sql | 1 + ...stream_name___conflict_stream_name_ab1.sql | 7 +- ...stream_name___conflict_stream_name_ab2.sql | 5 +- ...stream_name___conflict_stream_name_ab3.sql | 1 + .../conflict_stream_name_ab1.sql | 5 +- .../conflict_stream_name_ab2.sql | 5 +- .../conflict_stream_name_ab3.sql | 1 + ...t_stream_name_conflict_stream_name_ab1.sql | 7 +- ...t_stream_name_conflict_stream_name_ab2.sql | 5 +- ...t_stream_name_conflict_stream_name_ab3.sql | 1 + .../conflict_stream_scalar_ab1.sql | 5 +- .../conflict_stream_scalar_ab2.sql | 5 +- .../conflict_stream_scalar_ab3.sql | 1 + ...ream_with_c___long_names_partition_ab1.sql | 7 +- ...ream_with_c___long_names_partition_ab2.sql | 5 +- ...ream_with_c___long_names_partition_ab3.sql | 1 + ...ream_with_c___names_partition_data_ab1.sql | 9 +- ...ream_with_c___names_partition_data_ab2.sql | 5 +- ...ream_with_c___names_partition_data_ab3.sql | 1 + ...ream_with_c__column___with__quotes_ab1.sql | 9 +- ...ream_with_c__column___with__quotes_ab2.sql | 5 +- ...ream_with_c__column___with__quotes_ab3.sql | 1 + ...ream_with_c__ion_double_array_data_ab1.sql | 9 +- ...ream_with_c__ion_double_array_data_ab2.sql | 5 +- ...ream_with_c__ion_double_array_data_ab3.sql | 1 + ...ream_with_c__lting_into_long_names_ab1.sql | 6 +- ...ream_with_c__lting_into_long_names_ab2.sql | 6 +- ...d_stream_wi__lting_into_long_names_ab1.sql | 5 +- ...d_stream_wi__lting_into_long_names_ab2.sql | 5 +- ...d_stream_wi__lting_into_long_names_ab3.sql | 1 + .../test_normalization/unnest_alias_ab1.sql | 6 +- .../test_normalization/unnest_alias_ab2.sql | 6 +- .../test_normalization/unnest_alias_ab3.sql | 2 + .../unnest_alias_children_ab1.sql | 9 +- .../unnest_alias_children_ab2.sql | 5 +- .../unnest_alias_children_ab3.sql | 1 + .../unnest_alias_children_owner_ab1.sql | 7 +- .../unnest_alias_children_owner_ab2.sql | 5 +- .../unnest_alias_children_owner_ab3.sql | 1 + ...ream_with_n__lting_into_long_names_ab1.sql | 5 +- ...ream_with_n__lting_into_long_names_ab2.sql | 5 +- ...ream_with_n__lting_into_long_names_ab3.sql | 1 + ...ream_with_c__lting_into_long_names_scd.sql | 81 ++++++++++ ...d_stream_with_c__lting_into_long_names.sql | 13 +- .../test_normalization/unnest_alias.sql | 10 +- .../conflict_stream_array.sql | 3 + .../conflict_stream_name.sql | 3 + ...ict_stream_name___conflict_stream_name.sql | 3 + ...flict_stream_name_conflict_stream_name.sql | 3 + .../conflict_stream_scalar.sql | 3 + ...d_stream_with_c___long_names_partition.sql | 3 + ...d_stream_with_c___names_partition_data.sql | 3 + ...d_stream_with_c__column___with__quotes.sql | 3 + ...d_stream_with_c__ion_double_array_data.sql | 3 + ...ested_stream_wi__lting_into_long_names.sql | 3 + .../unnest_alias_children.sql | 3 + .../unnest_alias_children_owner.sql | 3 + ...e_stream_with_n__lting_into_long_names.sql | 3 + ...ream_with_c__lting_into_long_names_ab3.sql | 2 + .../tmp_test_data_check_row_counts.sql | 19 --- .../conflict_stream_array_ab1.sql | 12 +- .../conflict_stream_array_ab2.sql | 12 +- .../conflict_stream_array_ab3.sql | 8 +- ...stream_name___conflict_stream_name_ab1.sql | 14 +- ...stream_name___conflict_stream_name_ab2.sql | 12 +- ...stream_name___conflict_stream_name_ab3.sql | 8 +- .../conflict_stream_name_ab1.sql | 12 +- .../conflict_stream_name_ab2.sql | 12 +- .../conflict_stream_name_ab3.sql | 8 +- ...t_stream_name_conflict_stream_name_ab1.sql | 14 +- ...t_stream_name_conflict_stream_name_ab2.sql | 12 +- ...t_stream_name_conflict_stream_name_ab3.sql | 8 +- .../conflict_stream_scalar_ab1.sql | 12 +- .../conflict_stream_scalar_ab2.sql | 12 +- .../conflict_stream_scalar_ab3.sql | 8 +- ...ream_with_c___long_names_partition_ab1.sql | 14 +- ...ream_with_c___long_names_partition_ab2.sql | 12 +- ...ream_with_c___long_names_partition_ab3.sql | 8 +- ...ream_with_c___names_partition_data_ab1.sql | 16 +- ...ream_with_c___names_partition_data_ab2.sql | 12 +- ...ream_with_c___names_partition_data_ab3.sql | 8 +- ...ream_with_c__column___with__quotes_ab1.sql | 16 +- ...ream_with_c__column___with__quotes_ab2.sql | 12 +- ...ream_with_c__column___with__quotes_ab3.sql | 8 +- ...ream_with_c__ion_double_array_data_ab1.sql | 16 +- ...ream_with_c__ion_double_array_data_ab2.sql | 12 +- ...ream_with_c__ion_double_array_data_ab3.sql | 8 +- ...ream_with_c__lting_into_long_names_ab1.sql | 13 +- ...ream_with_c__lting_into_long_names_ab2.sql | 13 +- ...ream_with_c__lting_into_long_names_ab4.sql | 11 -- ...d_stream_wi__lting_into_long_names_ab1.sql | 12 +- ...d_stream_wi__lting_into_long_names_ab2.sql | 12 +- ...d_stream_wi__lting_into_long_names_ab3.sql | 8 +- .../test_normalization/unnest_alias_ab1.sql | 13 +- .../test_normalization/unnest_alias_ab2.sql | 13 +- .../test_normalization/unnest_alias_ab3.sql | 9 +- .../unnest_alias_children_ab1.sql | 16 +- .../unnest_alias_children_ab2.sql | 12 +- .../unnest_alias_children_ab3.sql | 8 +- .../unnest_alias_children_owner_ab1.sql | 14 +- .../unnest_alias_children_owner_ab2.sql | 12 +- .../unnest_alias_children_owner_ab3.sql | 8 +- ...ream_with_n__lting_into_long_names_ab1.sql | 12 +- ...ream_with_n__lting_into_long_names_ab2.sql | 12 +- ...ream_with_n__lting_into_long_names_ab3.sql | 8 +- ...ream_with_c__lting_into_long_names_scd.sql | 108 +++++++++++++ ...d_stream_with_c__lting_into_long_names.sql | 14 +- .../test_normalization/unnest_alias.sql | 19 +++ .../conflict_stream_array.sql | 10 +- .../conflict_stream_name.sql | 10 +- ...ict_stream_name___conflict_stream_name.sql | 10 +- ...flict_stream_name_conflict_stream_name.sql | 10 +- .../conflict_stream_scalar.sql | 10 +- ...d_stream_with_c___long_names_partition.sql | 10 +- ...d_stream_with_c___names_partition_data.sql | 10 +- ...d_stream_with_c__column___with__quotes.sql | 10 +- ...d_stream_with_c__ion_double_array_data.sql | 10 +- ...ream_with_c__lting_into_long_names_scd.sql | 21 --- ...ested_stream_wi__lting_into_long_names.sql | 10 +- .../test_normalization/unnest_alias.sql | 10 -- .../unnest_alias_children.sql | 10 +- .../unnest_alias_children_owner.sql | 10 +- ...e_stream_with_n__lting_into_long_names.sql | 10 +- ...ream_with_c__lting_into_long_names_ab3.sql | 9 +- .../conflict_stream_array_ab1.sql | 14 ++ .../conflict_stream_array_ab2.sql | 16 ++ .../conflict_stream_array_ab3.sql | 17 ++ ...stream_name___conflict_stream_name_ab1.sql | 15 ++ ...stream_name___conflict_stream_name_ab2.sql | 16 ++ ...stream_name___conflict_stream_name_ab3.sql | 17 ++ .../conflict_stream_name_ab1.sql | 16 ++ .../conflict_stream_name_ab2.sql | 18 +++ .../conflict_stream_name_ab3.sql | 17 ++ ...t_stream_name_conflict_stream_name_ab1.sql | 17 ++ ...t_stream_name_conflict_stream_name_ab2.sql | 16 ++ ...t_stream_name_conflict_stream_name_ab3.sql | 17 ++ .../conflict_stream_scalar_ab1.sql | 14 ++ .../conflict_stream_scalar_ab2.sql | 18 +++ .../conflict_stream_scalar_ab3.sql | 17 ++ ...ream_with_c___long_names_partition_ab1.sql | 17 ++ ...ream_with_c___long_names_partition_ab2.sql | 16 ++ ...ream_with_c___long_names_partition_ab3.sql | 21 +++ ...ream_with_c___names_partition_data_ab1.sql | 21 +++ ...ream_with_c___names_partition_data_ab2.sql | 16 ++ ...ream_with_c___names_partition_data_ab3.sql | 17 ++ ...ream_with_c__column___with__quotes_ab1.sql | 21 +++ ...ream_with_c__column___with__quotes_ab2.sql | 16 ++ ...ream_with_c__column___with__quotes_ab3.sql | 17 ++ ...ream_with_c__ion_double_array_data_ab1.sql | 21 +++ ...ream_with_c__ion_double_array_data_ab2.sql | 16 ++ ...ream_with_c__ion_double_array_data_ab3.sql | 17 ++ ...ream_with_c__lting_into_long_names_ab1.sql | 18 +++ ...ream_with_c__lting_into_long_names_ab2.sql | 22 +++ ...d_stream_wi__lting_into_long_names_ab1.sql | 14 ++ ...d_stream_wi__lting_into_long_names_ab2.sql | 18 +++ ...d_stream_wi__lting_into_long_names_ab3.sql | 17 ++ .../test_normalization/unnest_alias_ab1.sql | 15 ++ .../test_normalization/unnest_alias_ab2.sql | 17 ++ .../test_normalization/unnest_alias_ab3.sql | 18 +++ .../unnest_alias_children_ab1.sql | 24 +++ .../unnest_alias_children_ab2.sql | 19 +++ .../unnest_alias_children_ab3.sql | 19 +++ .../unnest_alias_children_owner_ab1.sql | 15 ++ .../unnest_alias_children_owner_ab2.sql | 16 ++ .../unnest_alias_children_owner_ab3.sql | 17 ++ ...ream_with_n__lting_into_long_names_ab1.sql | 14 ++ ...ream_with_n__lting_into_long_names_ab2.sql | 18 +++ ...ream_with_n__lting_into_long_names_ab3.sql | 17 ++ ...ream_with_c__lting_into_long_names_scd.sql | 14 ++ ...d_stream_with_c__lting_into_long_names.sql | 14 ++ .../test_normalization/unnest_alias.sql | 14 ++ .../conflict_stream_array.sql | 17 ++ .../conflict_stream_name.sql | 17 ++ ...ict_stream_name___conflict_stream_name.sql | 17 ++ ...flict_stream_name_conflict_stream_name.sql | 17 ++ .../conflict_stream_scalar.sql | 17 ++ ...d_stream_with_c___long_names_partition.sql | 19 +++ ...d_stream_with_c___names_partition_data.sql | 17 ++ ...d_stream_with_c__column___with__quotes.sql | 17 ++ ...d_stream_with_c__ion_double_array_data.sql | 17 ++ ...ested_stream_wi__lting_into_long_names.sql | 17 ++ .../unnest_alias_children.sql | 18 +++ .../unnest_alias_children_owner.sql | 17 ++ ...e_stream_with_n__lting_into_long_names.sql | 17 ++ ...ream_with_c__lting_into_long_names_ab3.sql | 20 +++ .../dedup_cdc_excluded_ab4.sql | 13 -- .../dedup_exchange_rate_ab4.sql | 13 -- .../test_normalization/pos_dedup_cdcx_ab4.sql | 13 -- .../dedup_cdc_excluded_scd.sql | 28 ---- .../dedup_exchange_rate_scd.sql | 34 ---- .../test_normalization/pos_dedup_cdcx_scd.sql | 28 ---- .../dedup_cdc_excluded_ab1.sql | 7 +- .../dedup_cdc_excluded_ab2.sql | 9 +- .../dedup_exchange_rate_ab1.sql | 6 +- .../dedup_exchange_rate_ab2.sql | 6 +- .../test_normalization/exchange_rate_ab1.sql | 6 +- .../test_normalization/exchange_rate_ab2.sql | 6 +- .../test_normalization/exchange_rate_ab3.sql | 2 + .../test_normalization/pos_dedup_cdcx_ab1.sql | 5 +- .../test_normalization/pos_dedup_cdcx_ab2.sql | 5 +- .../renamed_dedup_cdc_excluded_ab1.sql | 14 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 16 ++ .../dedup_cdc_excluded_scd.sql | 93 +++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 77 +++++++++ .../test_normalization/dedup_cdc_excluded.sql | 14 +- .../dedup_exchange_rate.sql | 13 +- .../test_normalization/exchange_rate.sql | 1 + .../renamed_dedup_cdc_excluded.sql | 21 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 97 ++++++++++++ .../test_normalization/pos_dedup_cdcx.sql | 6 +- .../dedup_cdc_excluded_ab3.sql | 4 +- .../dedup_exchange_rate_ab3.sql | 2 + .../test_normalization/pos_dedup_cdcx_ab3.sql | 1 + .../renamed_dedup_cdc_excluded_ab3.sql | 16 ++ .../tmp_test_data_check_row_counts.sql | 43 ----- .../dedup_cdc_excluded_ab1.sql | 14 +- .../dedup_cdc_excluded_ab2.sql | 14 +- .../dedup_cdc_excluded_ab4.sql | 11 -- .../dedup_exchange_rate_ab1.sql | 13 +- .../dedup_exchange_rate_ab2.sql | 13 +- .../dedup_exchange_rate_ab4.sql | 11 -- .../test_normalization/exchange_rate_ab1.sql | 6 +- .../test_normalization/exchange_rate_ab2.sql | 6 +- .../test_normalization/exchange_rate_ab3.sql | 3 + .../test_normalization/pos_dedup_cdcx_ab1.sql | 12 +- .../test_normalization/pos_dedup_cdcx_ab2.sql | 12 +- .../test_normalization/pos_dedup_cdcx_ab4.sql | 11 -- .../renamed_dedup_cdc_excluded_ab1.sql | 17 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 17 ++ .../dedup_cdc_excluded_scd.sql | 112 +++++++++++++ .../dedup_exchange_rate_scd.sql | 148 ++++++++++++++---- .../renamed_dedup_cdc_excluded_scd.sql | 104 ++++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 24 +++ .../dedup_exchange_rate.sql | 4 +- .../test_normalization/exchange_rate.sql | 4 +- .../renamed_dedup_cdc_excluded.sql | 20 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 114 ++++++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 15 -- .../dedup_cdc_excluded_scd.sql | 24 --- .../dedup_exchange_rate.sql | 17 -- .../dedup_exchange_rate_scd.sql | 26 --- .../test_normalization/pos_dedup_cdcx.sql | 13 +- .../test_normalization/pos_dedup_cdcx_scd.sql | 24 --- .../dedup_cdc_excluded_ab3.sql | 10 +- .../dedup_exchange_rate_ab3.sql | 9 +- .../test_normalization/pos_dedup_cdcx_ab3.sql | 8 +- .../renamed_dedup_cdc_excluded_ab3.sql | 17 ++ .../models/generated/sources.yml | 1 + .../dedup_cdc_excluded_ab1.sql | 18 +++ .../dedup_cdc_excluded_ab2.sql | 28 ++++ .../dedup_exchange_rate_ab1.sql | 21 +++ .../dedup_exchange_rate_ab2.sql | 37 +++++ .../test_normalization/exchange_rate_ab1.sql | 21 +++ .../test_normalization/exchange_rate_ab2.sql | 37 +++++ .../test_normalization/exchange_rate_ab3.sql | 30 ++++ .../test_normalization/pos_dedup_cdcx_ab1.sql | 18 +++ .../test_normalization/pos_dedup_cdcx_ab2.sql | 30 ++++ .../renamed_dedup_cdc_excluded_ab1.sql | 14 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 16 ++ .../dedup_cdc_excluded_scd.sql | 14 ++ .../renamed_dedup_cdc_excluded_scd.sql | 14 ++ .../test_normalization/dedup_cdc_excluded.sql | 14 ++ .../dedup_exchange_rate.sql | 14 ++ .../test_normalization/exchange_rate.sql | 4 +- .../renamed_dedup_cdc_excluded.sql | 14 ++ .../test_normalization/pos_dedup_cdcx_scd.sql | 97 ++++++++++++ .../test_normalization/pos_dedup_cdcx.sql | 23 +++ .../dedup_cdc_excluded_ab3.sql | 24 +++ .../dedup_exchange_rate_ab3.sql | 30 ++++ .../test_normalization/pos_dedup_cdcx_ab3.sql | 25 +++ .../renamed_dedup_cdc_excluded_ab3.sql | 16 ++ .../dedup_exchange_rate_ab1.sql | 21 +++ .../dedup_exchange_rate_ab2.sql | 37 +++++ .../test_normalization/exchange_rate_ab1.sql | 21 +++ .../test_normalization/exchange_rate_ab2.sql | 37 +++++ .../test_normalization/exchange_rate_ab3.sql | 30 ++++ .../renamed_dedup_cdc_excluded_ab1.sql | 18 +++ .../renamed_dedup_cdc_excluded_ab2.sql | 28 ++++ .../renamed_dedup_cdc_excluded_scd.sql | 14 ++ .../dedup_exchange_rate.sql | 14 ++ .../test_normalization/exchange_rate.sql | 14 ++ .../renamed_dedup_cdc_excluded.sql | 14 ++ .../dedup_exchange_rate_ab3.sql | 30 ++++ .../renamed_dedup_cdc_excluded_ab3.sql | 24 +++ ..._columns_resulting_into_long_names_scd.sql | 69 -------- ..._columns_resulting_into_long_names_scd.sql | 73 +++++++++ ...plex_columns_resulting_into_long_names.sql | 14 +- .../test_normalization/unnest_alias.sql | 25 ++- .../conflict_stream_array.sql | 15 +- .../conflict_stream_name.sql | 15 +- ...flict_stream_name_conflict_stream_name.sql | 17 +- ...flict_stream_name_conflict_stream_name.sql | 17 +- .../conflict_stream_scalar.sql | 15 +- ...ns_resulting_into_long_names_partition.sql | 17 +- ..._names_partition_column___with__quotes.sql | 19 ++- ...sulting_into_long_names_partition_data.sql | 19 ++- ...long_names_partition_double_array_data.sql | 19 ++- ...ut_namespace_resulting_into_long_names.sql | 15 +- .../unnest_alias_children.sql | 19 ++- .../unnest_alias_children_owner.sql | 17 +- ...th_namespace_resulting_into_long_names.sql | 15 +- ..._columns_resulting_into_long_names_ab3.sql | 43 +++++ .../tmp_test_data_check_row_counts.sql | 19 --- .../conflict_stream_array_ab1.sql | 12 +- .../conflict_stream_array_ab2.sql | 12 +- .../conflict_stream_array_ab3.sql | 8 +- .../conflict_stream_name_ab1.sql | 12 +- .../conflict_stream_name_ab2.sql | 12 +- .../conflict_stream_name_ab3.sql | 8 +- ...t_stream_name_conflict_stream_name_ab1.sql | 14 +- ...t_stream_name_conflict_stream_name_ab2.sql | 12 +- ...t_stream_name_conflict_stream_name_ab3.sql | 8 +- ...t_stream_name_conflict_stream_name_ab1.sql | 14 +- ...t_stream_name_conflict_stream_name_ab2.sql | 12 +- ...t_stream_name_conflict_stream_name_ab3.sql | 8 +- .../conflict_stream_scalar_ab1.sql | 12 +- .../conflict_stream_scalar_ab2.sql | 12 +- .../conflict_stream_scalar_ab3.sql | 8 +- ..._columns_resulting_into_long_names_ab1.sql | 13 +- ..._columns_resulting_into_long_names_ab2.sql | 13 +- ..._columns_resulting_into_long_names_ab4.sql | 11 -- ...esulting_into_long_names_partition_ab1.sql | 14 +- ...esulting_into_long_names_partition_ab2.sql | 12 +- ...esulting_into_long_names_partition_ab3.sql | 8 +- ...es_partition_column___with__quotes_ab1.sql | 16 +- ...es_partition_column___with__quotes_ab2.sql | 12 +- ...es_partition_column___with__quotes_ab3.sql | 8 +- ...ing_into_long_names_partition_data_ab1.sql | 16 +- ...ing_into_long_names_partition_data_ab2.sql | 12 +- ...ing_into_long_names_partition_data_ab3.sql | 8 +- ..._names_partition_double_array_data_ab1.sql | 16 +- ..._names_partition_double_array_data_ab2.sql | 12 +- ..._names_partition_double_array_data_ab3.sql | 8 +- ...amespace_resulting_into_long_names_ab1.sql | 12 +- ...amespace_resulting_into_long_names_ab2.sql | 12 +- ...amespace_resulting_into_long_names_ab3.sql | 8 +- .../test_normalization/unnest_alias_ab1.sql | 13 +- .../test_normalization/unnest_alias_ab2.sql | 13 +- .../test_normalization/unnest_alias_ab3.sql | 9 +- .../unnest_alias_children_ab1.sql | 16 +- .../unnest_alias_children_ab2.sql | 12 +- .../unnest_alias_children_ab3.sql | 8 +- .../unnest_alias_children_owner_ab1.sql | 14 +- .../unnest_alias_children_owner_ab2.sql | 12 +- .../unnest_alias_children_owner_ab3.sql | 8 +- ...amespace_resulting_into_long_names_ab1.sql | 12 +- ...amespace_resulting_into_long_names_ab2.sql | 12 +- ...amespace_resulting_into_long_names_ab3.sql | 8 +- ..._columns_resulting_into_long_names_scd.sql | 108 +++++++++++++ ...plex_columns_resulting_into_long_names.sql | 14 +- .../test_normalization/unnest_alias.sql | 19 +++ .../conflict_stream_array.sql | 10 +- .../conflict_stream_name.sql | 10 +- ...flict_stream_name_conflict_stream_name.sql | 10 +- ...flict_stream_name_conflict_stream_name.sql | 10 +- .../conflict_stream_scalar.sql | 10 +- ...ns_resulting_into_long_names_partition.sql | 10 +- ..._names_partition_column___with__quotes.sql | 10 +- ...sulting_into_long_names_partition_data.sql | 10 +- ...long_names_partition_double_array_data.sql | 10 +- ..._columns_resulting_into_long_names_scd.sql | 21 --- ...ut_namespace_resulting_into_long_names.sql | 10 +- .../test_normalization/unnest_alias.sql | 10 -- .../unnest_alias_children.sql | 10 +- .../unnest_alias_children_owner.sql | 10 +- ...th_namespace_resulting_into_long_names.sql | 10 +- ..._columns_resulting_into_long_names_ab3.sql | 9 +- ..._columns_resulting_into_long_names_scd.sql | 14 ++ ...plex_columns_resulting_into_long_names.sql | 14 ++ .../test_normalization/unnest_alias.sql | 14 ++ .../conflict_stream_array.sql | 54 +++++++ .../conflict_stream_name.sql | 56 +++++++ ...flict_stream_name_conflict_stream_name.sql | 57 +++++++ ...flict_stream_name_conflict_stream_name.sql | 55 +++++++ .../conflict_stream_scalar.sql | 56 +++++++ ...ns_resulting_into_long_names_partition.sql | 61 ++++++++ ..._names_partition_column___with__quotes.sql | 100 ++++++++++++ ...sulting_into_long_names_partition_data.sql | 100 ++++++++++++ ...long_names_partition_double_array_data.sql | 100 ++++++++++++ ...ut_namespace_resulting_into_long_names.sql | 54 +++++++ .../unnest_alias_children.sql | 107 +++++++++++++ .../unnest_alias_children_owner.sql | 57 +++++++ ...th_namespace_resulting_into_long_names.sql | 54 +++++++ ..._columns_resulting_into_long_names_ab3.sql | 43 +++++ .../dedup_cdc_excluded_scd.sql | 84 ---------- .../dedup_cdc_excluded_scd.sql | 77 +++++++++ .../dedup_exchange_rate_scd.sql | 83 ++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 69 ++++++++ .../test_normalization/dedup_cdc_excluded.sql | 15 +- .../dedup_exchange_rate.sql | 14 +- .../test_normalization/exchange_rate.sql | 25 ++- .../renamed_dedup_cdc_excluded.sql | 25 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 77 +++++++++ .../test_normalization/pos_dedup_cdcx.sql | 7 +- .../dedup_cdc_excluded_ab3.sql | 53 +++++++ .../dedup_exchange_rate_ab3.sql} | 57 ++----- .../pos_dedup_cdcx_ab3.sql} | 56 ++----- .../renamed_dedup_cdc_excluded_ab3.sql | 39 +++++ .../tmp_test_data_check_row_counts.sql | 43 ----- .../dedup_cdc_excluded_ab1.sql | 14 +- .../dedup_cdc_excluded_ab2.sql | 14 +- .../dedup_cdc_excluded_ab3.sql | 15 -- .../dedup_cdc_excluded_ab4.sql | 11 -- .../dedup_exchange_rate_ab1.sql | 13 +- .../dedup_exchange_rate_ab2.sql | 13 +- .../dedup_exchange_rate_ab4.sql | 11 -- .../test_normalization/exchange_rate_ab1.sql | 13 +- .../test_normalization/exchange_rate_ab2.sql | 13 +- .../test_normalization/exchange_rate_ab3.sql | 9 +- .../test_normalization/pos_dedup_cdcx_ab1.sql | 12 +- .../test_normalization/pos_dedup_cdcx_ab2.sql | 12 +- .../test_normalization/pos_dedup_cdcx_ab4.sql | 11 -- .../renamed_dedup_cdc_excluded_ab1.sql | 17 ++ .../renamed_dedup_cdc_excluded_ab2.sql | 17 ++ .../dedup_cdc_excluded_scd.sql | 112 +++++++++++++ .../dedup_exchange_rate_scd.sql | 122 +++++++++++++++ .../renamed_dedup_cdc_excluded_scd.sql | 104 ++++++++++++ .../test_normalization/dedup_cdc_excluded.sql | 24 +++ .../dedup_exchange_rate.sql | 14 +- .../test_normalization/exchange_rate.sql | 11 +- .../renamed_dedup_cdc_excluded.sql | 20 +++ .../test_normalization/pos_dedup_cdcx_scd.sql | 114 ++++++++++++++ .../dedup_cdc_excluded_scd.sql | 24 --- .../dedup_exchange_rate_scd.sql | 26 --- .../test_normalization/pos_dedup_cdcx.sql | 13 +- .../test_normalization/pos_dedup_cdcx_scd.sql | 24 --- .../dedup_cdc_excluded_ab3.sql | 10 +- .../dedup_exchange_rate_ab3.sql | 9 +- .../test_normalization/pos_dedup_cdcx_ab3.sql | 8 +- .../renamed_dedup_cdc_excluded_ab3.sql | 17 ++ .../models/generated/sources.yml | 1 + .../dedup_cdc_excluded_scd.sql | 14 ++ .../dedup_exchange_rate_scd.sql | 14 ++ .../renamed_dedup_cdc_excluded_scd.sql | 14 ++ .../test_normalization/dedup_cdc_excluded.sql | 14 ++ .../dedup_exchange_rate.sql | 14 ++ .../test_normalization/exchange_rate.sql | 14 ++ .../renamed_dedup_cdc_excluded.sql | 14 ++ .../test_normalization/pos_dedup_cdcx_scd.sql | 77 +++++++++ .../test_normalization/pos_dedup_cdcx.sql | 27 ++++ .../dedup_cdc_excluded_ab3.sql | 53 +++++++ .../dedup_exchange_rate_ab3.sql | 63 ++++++++ .../test_normalization/pos_dedup_cdcx_ab3.sql | 54 +++++++ .../renamed_dedup_cdc_excluded_ab3.sql | 39 +++++ .../dedup_exchange_rate_scd.sql | 14 ++ .../renamed_dedup_cdc_excluded_scd.sql | 14 ++ .../dedup_exchange_rate.sql | 14 ++ .../test_normalization/exchange_rate.sql | 14 ++ .../renamed_dedup_cdc_excluded.sql | 14 ++ .../dedup_exchange_rate_ab3.sql | 65 ++++++++ .../renamed_dedup_cdc_excluded_ab3.sql | 53 +++++++ ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql | 13 -- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql | 25 --- .../CONFLICT_STREAM_ARRAY_AB1.sql | 5 +- .../CONFLICT_STREAM_ARRAY_AB2.sql | 5 +- .../CONFLICT_STREAM_ARRAY_AB3.sql | 1 + .../CONFLICT_STREAM_NAME_AB1.sql | 5 +- .../CONFLICT_STREAM_NAME_AB2.sql | 5 +- .../CONFLICT_STREAM_NAME_AB3.sql | 1 + ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql | 7 +- ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql | 5 +- ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql | 1 + ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql | 7 +- ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql | 5 +- ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql | 1 + .../CONFLICT_STREAM_SCALAR_AB1.sql | 5 +- .../CONFLICT_STREAM_SCALAR_AB2.sql | 5 +- .../CONFLICT_STREAM_SCALAR_AB3.sql | 1 + ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql | 6 +- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql | 6 +- ...ESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql | 7 +- ...ESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql | 5 +- ...ESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql | 1 + ...ES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql | 9 +- ...ES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql | 5 +- ...ES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql | 1 + ...ING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql | 9 +- ...ING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql | 5 +- ...ING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql | 1 + ..._NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql | 9 +- ..._NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql | 5 +- ..._NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql | 1 + ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql | 5 +- ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql | 5 +- ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql | 1 + .../TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql | 6 +- .../TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql | 6 +- .../TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql | 2 + .../UNNEST_ALIAS_CHILDREN_AB1.sql | 9 +- .../UNNEST_ALIAS_CHILDREN_AB2.sql | 5 +- .../UNNEST_ALIAS_CHILDREN_AB3.sql | 1 + .../UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql | 7 +- .../UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql | 5 +- .../UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql | 1 + ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql | 5 +- ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql | 5 +- ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql | 1 + ...PLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql | 13 +- .../TEST_NORMALIZATION/UNNEST_ALIAS.sql | 10 +- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql | 81 ++++++++++ .../CONFLICT_STREAM_ARRAY.sql | 9 +- .../CONFLICT_STREAM_NAME.sql | 9 +- ...FLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql | 9 +- ...FLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql | 9 +- .../CONFLICT_STREAM_SCALAR.sql | 9 +- ...NS_RESULTING_INTO_LONG_NAMES_PARTITION.sql | 9 +- ..._NAMES_PARTITION_COLUMN___WITH__QUOTES.sql | 9 +- ...SULTING_INTO_LONG_NAMES_PARTITION_DATA.sql | 9 +- ...LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql | 9 +- ...UT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql | 9 +- .../UNNEST_ALIAS_CHILDREN.sql | 9 +- .../UNNEST_ALIAS_CHILDREN_OWNER.sql | 9 +- ...TH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql | 9 +- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql | 2 + .../tmp_test_data_check_row_counts.sql | 19 --- .../CONFLICT_STREAM_ARRAY_AB1.sql | 12 +- .../CONFLICT_STREAM_ARRAY_AB2.sql | 12 +- .../CONFLICT_STREAM_ARRAY_AB3.sql | 8 +- .../CONFLICT_STREAM_NAME_AB1.sql | 12 +- .../CONFLICT_STREAM_NAME_AB2.sql | 12 +- .../CONFLICT_STREAM_NAME_AB3.sql | 8 +- ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql | 14 +- ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql | 12 +- ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql | 8 +- ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql | 14 +- ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql | 12 +- ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql | 8 +- .../CONFLICT_STREAM_SCALAR_AB1.sql | 12 +- .../CONFLICT_STREAM_SCALAR_AB2.sql | 12 +- .../CONFLICT_STREAM_SCALAR_AB3.sql | 8 +- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql | 13 +- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql | 13 +- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql | 11 -- ...ESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql | 14 +- ...ESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql | 12 +- ...ESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql | 8 +- ...ES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql | 16 +- ...ES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql | 12 +- ...ES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql | 8 +- ...ING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql | 16 +- ...ING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql | 12 +- ...ING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql | 8 +- ..._NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql | 16 +- ..._NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql | 12 +- ..._NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql | 8 +- ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql | 12 +- ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql | 12 +- ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql | 8 +- .../TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql | 13 +- .../TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql | 13 +- .../TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql | 9 +- .../UNNEST_ALIAS_CHILDREN_AB1.sql | 16 +- .../UNNEST_ALIAS_CHILDREN_AB2.sql | 12 +- .../UNNEST_ALIAS_CHILDREN_AB3.sql | 8 +- .../UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql | 14 +- .../UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql | 12 +- .../UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql | 8 +- ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql | 12 +- ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql | 12 +- ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql | 8 +- ...PLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql | 14 +- .../TEST_NORMALIZATION/UNNEST_ALIAS.sql | 19 +++ ..._COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql | 108 +++++++++++++ .../CONFLICT_STREAM_ARRAY.sql | 10 +- .../CONFLICT_STREAM_NAME.sql | 10 +- ...FLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql | 10 +- ...FLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql | 10 +- .../CONFLICT_STREAM_SCALAR.sql | 10 +- ...NS_RESULTING_INTO_LONG_NAMES_PARTITION.sql | 10 +- ..._NAMES_PARTITION_COLUMN___WITH__QUOTES.sql | 10 +- ...SULTING_INTO_LONG_NAMES_PARTITION_DATA.sql | 10 +- ...LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql | 10 +- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql | 21 --- ...UT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql | 10 +- .../TEST_NORMALIZATION/UNNEST_ALIAS.sql | 10 -- .../UNNEST_ALIAS_CHILDREN.sql | 10 +- .../UNNEST_ALIAS_CHILDREN_OWNER.sql | 10 +- ...TH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql | 10 +- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql | 9 +- .../CONFLICT_STREAM_ARRAY_AB1.sql | 14 ++ .../CONFLICT_STREAM_ARRAY_AB2.sql | 16 ++ .../CONFLICT_STREAM_ARRAY_AB3.sql | 17 ++ .../CONFLICT_STREAM_NAME_AB1.sql | 16 ++ .../CONFLICT_STREAM_NAME_AB2.sql | 18 +++ .../CONFLICT_STREAM_NAME_AB3.sql | 17 ++ ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql | 17 ++ ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql | 16 ++ ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql | 17 ++ ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql | 15 ++ ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql | 16 ++ ...T_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql | 17 ++ .../CONFLICT_STREAM_SCALAR_AB1.sql | 14 ++ .../CONFLICT_STREAM_SCALAR_AB2.sql | 18 +++ .../CONFLICT_STREAM_SCALAR_AB3.sql | 17 ++ ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql | 18 +++ ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql | 22 +++ ...ESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql | 17 ++ ...ESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql | 16 ++ ...ESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql | 21 +++ ...ES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql | 17 ++ ...ES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql | 16 ++ ...ES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql | 17 ++ ...ING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql | 17 ++ ...ING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql | 16 ++ ...ING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql | 17 ++ ..._NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql | 17 ++ ..._NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql | 16 ++ ..._NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql | 17 ++ ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql | 14 ++ ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql | 18 +++ ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql | 17 ++ .../TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql | 15 ++ .../TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql | 17 ++ .../TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql | 18 +++ .../UNNEST_ALIAS_CHILDREN_AB1.sql | 20 +++ .../UNNEST_ALIAS_CHILDREN_AB2.sql | 19 +++ .../UNNEST_ALIAS_CHILDREN_AB3.sql | 19 +++ .../UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql | 15 ++ .../UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql | 16 ++ .../UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql | 17 ++ ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql | 14 ++ ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql | 18 +++ ...AMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql | 17 ++ ...PLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql | 26 +++ .../TEST_NORMALIZATION/UNNEST_ALIAS.sql | 26 +++ ..._COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql | 26 +++ .../CONFLICT_STREAM_ARRAY.sql | 19 +++ .../CONFLICT_STREAM_NAME.sql | 19 +++ ...FLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql | 19 +++ ...FLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql | 19 +++ .../CONFLICT_STREAM_SCALAR.sql | 19 +++ ...NS_RESULTING_INTO_LONG_NAMES_PARTITION.sql | 21 +++ ..._NAMES_PARTITION_COLUMN___WITH__QUOTES.sql | 19 +++ ...SULTING_INTO_LONG_NAMES_PARTITION_DATA.sql | 19 +++ ...LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql | 19 +++ ...UT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql | 19 +++ .../UNNEST_ALIAS_CHILDREN.sql | 20 +++ .../UNNEST_ALIAS_CHILDREN_OWNER.sql | 19 +++ ...TH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql | 19 +++ ..._COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql | 20 +++ .../DEDUP_CDC_EXCLUDED_AB4.sql | 13 -- .../DEDUP_EXCHANGE_RATE_AB4.sql | 13 -- .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql | 13 -- .../DEDUP_CDC_EXCLUDED_SCD.sql | 28 ---- .../DEDUP_EXCHANGE_RATE_SCD.sql | 34 ---- .../TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql | 28 ---- .../DEDUP_CDC_EXCLUDED_AB1.sql | 7 +- .../DEDUP_CDC_EXCLUDED_AB2.sql | 9 +- .../DEDUP_EXCHANGE_RATE_AB1.sql | 6 +- .../DEDUP_EXCHANGE_RATE_AB2.sql | 6 +- .../TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql | 6 +- .../TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql | 6 +- .../TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql | 2 + .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql | 5 +- .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql | 5 +- .../RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql | 14 ++ .../RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql | 16 ++ .../TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql | 14 +- .../DEDUP_EXCHANGE_RATE.sql | 13 +- .../TEST_NORMALIZATION/EXCHANGE_RATE.sql | 10 +- .../RENAMED_DEDUP_CDC_EXCLUDED.sql | 21 +++ .../DEDUP_CDC_EXCLUDED_SCD.sql | 93 +++++++++++ .../DEDUP_EXCHANGE_RATE_SCD.sql | 99 ++++++++++++ .../RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql | 77 +++++++++ .../TEST_NORMALIZATION/POS_DEDUP_CDCX.sql | 12 +- .../TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql | 99 ++++++++++++ .../DEDUP_CDC_EXCLUDED_AB3.sql | 4 +- .../DEDUP_EXCHANGE_RATE_AB3.sql | 2 + .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql | 1 + .../RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql | 16 ++ .../tmp_test_data_check_row_counts.sql | 43 ----- .../DEDUP_CDC_EXCLUDED_AB1.sql | 14 +- .../DEDUP_CDC_EXCLUDED_AB2.sql | 14 +- .../DEDUP_CDC_EXCLUDED_AB4.sql | 11 -- .../DEDUP_EXCHANGE_RATE_AB1.sql | 13 +- .../DEDUP_EXCHANGE_RATE_AB2.sql | 13 +- .../DEDUP_EXCHANGE_RATE_AB4.sql | 11 -- .../TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql | 13 +- .../TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql | 13 +- .../TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql | 9 +- .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql | 12 +- .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql | 12 +- .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql | 11 -- .../RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql | 17 ++ .../RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql | 17 ++ .../TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql | 24 +++ .../DEDUP_EXCHANGE_RATE.sql | 14 +- .../TEST_NORMALIZATION/EXCHANGE_RATE.sql | 11 +- .../RENAMED_DEDUP_CDC_EXCLUDED.sql | 20 +++ .../DEDUP_CDC_EXCLUDED_SCD.sql | 112 +++++++++++++ .../DEDUP_EXCHANGE_RATE_SCD.sql | 122 +++++++++++++++ .../RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql | 104 ++++++++++++ .../TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql | 15 -- .../DEDUP_CDC_EXCLUDED_SCD.sql | 24 --- .../DEDUP_EXCHANGE_RATE_SCD.sql | 26 --- .../TEST_NORMALIZATION/POS_DEDUP_CDCX.sql | 13 +- .../TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql | 24 --- .../TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql | 114 ++++++++++++++ .../DEDUP_CDC_EXCLUDED_AB3.sql | 10 +- .../DEDUP_EXCHANGE_RATE_AB3.sql | 9 +- .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql | 8 +- .../RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql | 17 ++ .../models/generated/sources.yml | 1 + .../DEDUP_CDC_EXCLUDED_AB1.sql | 18 +++ .../DEDUP_CDC_EXCLUDED_AB2.sql | 28 ++++ .../DEDUP_EXCHANGE_RATE_AB1.sql | 21 +++ .../DEDUP_EXCHANGE_RATE_AB2.sql | 43 +++++ .../TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql | 21 +++ .../TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql | 43 +++++ .../TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql | 30 ++++ .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql | 18 +++ .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql | 30 ++++ .../RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql | 14 ++ .../RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql | 16 ++ .../TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql | 26 +++ .../DEDUP_EXCHANGE_RATE.sql | 26 +++ .../TEST_NORMALIZATION/EXCHANGE_RATE.sql | 26 +++ .../RENAMED_DEDUP_CDC_EXCLUDED.sql | 26 +++ .../DEDUP_CDC_EXCLUDED_SCD.sql | 26 +++ .../DEDUP_EXCHANGE_RATE_SCD.sql | 26 +++ .../RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql | 26 +++ .../TEST_NORMALIZATION/POS_DEDUP_CDCX.sql | 25 +++ .../TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql | 99 ++++++++++++ .../DEDUP_CDC_EXCLUDED_AB3.sql | 24 +++ .../DEDUP_EXCHANGE_RATE_AB3.sql | 30 ++++ .../TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql | 25 +++ .../RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql | 16 ++ 1631 files changed, 27131 insertions(+), 4421 deletions(-) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql (74%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql (93%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql (76%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql (93%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql (94%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql (76%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql (92%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql (75%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql (94%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql (91%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql (91%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab1.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab2.sql (74%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab3.sql (94%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql (76%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql (75%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql (91%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql (90%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql (77%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql (74%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql (89%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql (72%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/unnest_alias.sql (70%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_array.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql (77%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_scalar.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/unnest_alias_children.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/unnest_alias_children_owner.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql (89%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/{redshift/test_nested_streams/models/generated/airbyte_ctes => bigquery/test_nested_streams/models/generated/airbyte_views}/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql (53%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql (87%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab1.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab2.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab3.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql (86%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql (83%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_cdc_excluded.sql (66%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_exchange_rate.sql (68%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/exchange_rate.sql (74%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final => first_output}/airbyte_tables/test_normalization/pos_dedup_cdcx.sql (66%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/dedup_cdc_excluded_ab3.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/dedup_exchange_rate_ab3.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/pos_dedup_cdcx_ab3.sql (95%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/dedup_exchange_rate_ab3.sql (52%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/pos_dedup_cdcx_ab3.sql (52%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql (86%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql (86%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab1.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab2.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql (98%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/nested_stream_with_co__lting_into_long_names.sql (69%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/unnest_alias.sql (50%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_array.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_scalar.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/unnest_alias_children.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/unnest_alias_children_owner.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql (98%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/nested_stream_with_co__lting_into_long_names.sql (57%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql (61%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql (89%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab1.sql (89%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab2.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql (86%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_cdc_excluded.sql (52%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_exchange_rate.sql (54%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/exchange_rate.sql (53%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final => first_output}/airbyte_tables/test_normalization/pos_dedup_cdcx.sql (93%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/dedup_cdc_excluded_ab3.sql (91%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/dedup_exchange_rate_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/pos_dedup_cdcx_ab3.sql (98%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/{redshift/test_simple_streams/models/generated/airbyte_tables => mssql/test_simple_streams/models/generated/airbyte_incremental}/test_normalization/dedup_cdc_excluded.sql (50%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/dedup_exchange_rate.sql (57%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/{mysql/test_simple_streams/models/generated/airbyte_tables => mssql/test_simple_streams/models/generated/airbyte_incremental}/test_normalization/exchange_rate.sql (57%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/dedup_cdc_excluded_ab3.sql (60%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/{mysql/test_simple_streams/models/generated/airbyte_ctes => mssql/test_simple_streams/models/generated/airbyte_views}/test_normalization/dedup_exchange_rate_ab3.sql (65%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/{redshift/test_simple_streams/models/generated/airbyte_ctes => mssql/test_simple_streams/models/generated/airbyte_views}/test_normalization/pos_dedup_cdcx_ab3.sql (66%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql (75%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql (76%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql (77%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql (91%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql (91%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql (91%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab1.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab2.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab3.sql (96%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql (91%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql (76%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql (77%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql (97%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/nested_stream_with_co__lting_into_long_names.sql (75%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/unnest_alias.sql (77%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_array.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_scalar.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql (86%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/unnest_alias_children.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/unnest_alias_children_owner.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql (97%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/nested_stream_with_co__lting_into_long_names.sql (55%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql (61%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql (87%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab1.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab2.sql (86%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql (86%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql (83%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_cdc_excluded.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_exchange_rate.sql (76%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/exchange_rate.sql (82%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final => first_output}/airbyte_tables/test_normalization/pos_dedup_cdcx.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/dedup_cdc_excluded_ab3.sql (53%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/dedup_exchange_rate_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/pos_dedup_cdcx_ab3.sql (98%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/dedup_cdc_excluded.sql (52%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/dedup_exchange_rate.sql (57%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/{mssql/test_simple_streams/models/generated/airbyte_tables => mysql/test_simple_streams/models/generated/airbyte_incremental}/test_normalization/exchange_rate.sql (57%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/dedup_cdc_excluded_ab3.sql (61%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/{mssql/test_simple_streams/models/generated/airbyte_ctes => mysql/test_simple_streams/models/generated/airbyte_views}/test_normalization/dedup_exchange_rate_ab3.sql (65%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/pos_dedup_cdcx_ab3.sql (67%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_cdc_excluded.sql (72%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_exchange_rate.sql (74%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/exchange_rate.sql (88%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/{final => first_output}/airbyte_tables/test_normalization/pos_dedup_cdcx.sql (72%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/{final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql => first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql} (57%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/{final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql => first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql} (51%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/dedup_cdc_excluded.sql (50%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/dedup_exchange_rate.sql (51%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/exchange_rate.sql (53%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/dedup_cdc_excluded_ab3.sql (61%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/dedup_exchange_rate_ab3.sql (64%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/pos_dedup_cdcx_ab3.sql (68%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql (87%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab1.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab2.sql (77%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql (97%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/nested_stream_with_c__lting_into_long_names.sql (73%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/unnest_alias.sql (64%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_array.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql (87%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_scalar.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql (89%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql (87%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/unnest_alias_children.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/unnest_alias_children_owner.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql (97%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/nested_stream_with_c__lting_into_long_names.sql (52%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql (56%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql (90%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql (89%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab1.sql (90%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab2.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/exchange_rate_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql (89%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final => first_output}/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql (86%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_cdc_excluded.sql (74%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_exchange_rate.sql (76%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final => first_output}/airbyte_tables/test_normalization/pos_dedup_cdcx.sql (77%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/dedup_cdc_excluded_ab3.sql (89%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/dedup_exchange_rate_ab3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/test_normalization/pos_dedup_cdcx_ab3.sql (98%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/dedup_cdc_excluded_ab3.sql (55%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/dedup_exchange_rate_ab3.sql (60%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/pos_dedup_cdcx_ab3.sql (61%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql (61%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/unnest_alias.sql (76%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_array.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/conflict_stream_scalar.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql (90%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql (90%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql (90%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql (87%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/unnest_alias_children.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization/unnest_alias_children_owner.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/{final => first_output}/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql (86%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql (55%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/{bigquery/test_nested_streams/models/generated/airbyte_ctes => redshift/test_nested_streams/models/generated/airbyte_views}/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql (61%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_cdc_excluded.sql (57%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/dedup_exchange_rate.sql (61%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/test_normalization/exchange_rate.sql (88%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/{final => first_output}/airbyte_tables/test_normalization/pos_dedup_cdcx.sql (70%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/{final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql => first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql} (64%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/{final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql => first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql} (58%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/dedup_exchange_rate.sql (51%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/{airbyte_tables => airbyte_incremental}/test_normalization/exchange_rate.sql (53%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/{bigquery/test_simple_streams/models/generated/airbyte_ctes => redshift/test_simple_streams/models/generated/airbyte_views}/test_normalization/dedup_cdc_excluded_ab3.sql (56%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/test_normalization/dedup_exchange_rate_ab3.sql (61%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/{mssql/test_simple_streams/models/generated/airbyte_ctes => redshift/test_simple_streams/models/generated/airbyte_views}/test_normalization/pos_dedup_cdcx_ab3.sql (62%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql (74%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql (76%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql (77%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql (80%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql (76%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql (84%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql (72%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql (77%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql (75%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql (76%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql (97%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql (82%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql (79%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql (58%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final/airbyte_tables => first_output/airbyte_incremental}/TEST_NORMALIZATION/UNNEST_ALIAS.sql (56%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql (59%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql (58%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql (62%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql (64%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql (59%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql (67%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql (66%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql (64%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql (65%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql (64%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql (59%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql (59%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql (64%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/{final/airbyte_ctes => first_output/airbyte_views}/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql (98%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/{airbyte_tables => airbyte_incremental}/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql (53%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/{airbyte_ctes => airbyte_views}/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql (60%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql (85%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql (81%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql (91%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql (91%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql (86%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final => first_output}/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql (82%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql (53%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql (56%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final/airbyte_tables => first_output/airbyte_incremental}/TEST_NORMALIZATION/EXCHANGE_RATE.sql (62%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final => first_output}/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql (54%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql (90%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql (98%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/{final/airbyte_ctes => first_output/airbyte_views}/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql (98%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/{airbyte_tables => airbyte_incremental}/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql (51%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/{airbyte_tables => airbyte_incremental}/TEST_NORMALIZATION/EXCHANGE_RATE.sql (52%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql (55%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql (61%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/{airbyte_ctes => airbyte_views}/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql (61%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql deleted file mode 100644 index fe9e5f7b18ea..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab4` - OPTIONS() - as --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab3` tmp --- nested_stream_with_complex_columns_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql deleted file mode 100644 index 70229a37cad3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ /dev/null @@ -1,29 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` - - - OPTIONS() - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - date, - `partition`, - date as _airbyte_start_at, - lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab4` --- nested_stream_with_complex_columns_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index dae3ef07ec98..877a01b4c107 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -7,7 +7,10 @@ select json_extract_scalar(_airbyte_data, "$['id']") as id, json_extract_array(_airbyte_data, "$['conflict_stream_array']") as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array as table_alias --- conflict_stream_array; +-- conflict_stream_array +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql similarity index 74% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 43a66e1de59b..13acc814aa35 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -9,7 +9,10 @@ select string ) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` --- conflict_stream_array; +-- conflict_stream_array +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql similarity index 93% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 4854b7bdf8e2..b0a10f06f6fe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_conflict_stream_array_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` tmp --- conflict_stream_array; +-- conflict_stream_array +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index b2a5e728a659..e32dbb75a795 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -9,7 +9,10 @@ select json_extract(table_alias._airbyte_data, "$['conflict_stream_name']") as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name as table_alias --- conflict_stream_name; +-- conflict_stream_name +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index 0e880d8d1132..1e6066d334e2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -11,7 +11,10 @@ select cast(conflict_stream_name as string ) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` --- conflict_stream_name; +-- conflict_stream_name +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql similarity index 93% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index b1c04efa9773..f8867d6b5713 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_conflict_stream_name_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` tmp --- conflict_stream_name; +-- conflict_stream_name +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index 15ff7f7cc272..57cef056c421 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -9,8 +9,11 @@ select json_extract(table_alias.conflict_stream_name, "$['conflict_stream_name']") as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization.`conflict_stream_name` as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name; +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 17e8df654774..eae8466d7725 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -9,7 +9,10 @@ select cast(conflict_stream_name as string ) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name; +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index 66e0c7cc4189..0ea122e2ea01 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_conflict_stream_name_2_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name; +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql index 43ebdecf36c4..962511141a3b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql @@ -7,8 +7,11 @@ select _airbyte_conflict_stream_name_2_hashid, json_extract_scalar(conflict_stream_name, "$['groups']") as `groups`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias -where conflict_stream_name is not null --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name; +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql index b1674b6fb1cd..be894b9a62a3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql @@ -9,7 +9,10 @@ select cast(`groups` as string ) as `groups`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name; +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql similarity index 94% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql index 1fdb660bcfac..c468c76f6378 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_conflict_stream_name_3_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` tmp --- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name; +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index bd1d66a5840f..c933d2c41fac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -7,7 +7,10 @@ select json_extract_scalar(_airbyte_data, "$['id']") as id, json_extract_scalar(_airbyte_data, "$['conflict_stream_scalar']") as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar as table_alias --- conflict_stream_scalar; +-- conflict_stream_scalar +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index 2a3441329550..f1c973037920 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -11,7 +11,10 @@ select cast(conflict_stream_scalar as int64 ) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` --- conflict_stream_scalar; +-- conflict_stream_scalar +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql similarity index 92% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 5aea888837e8..2679d90727ee 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_conflict_stream_scalar_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp --- conflict_stream_scalar; +-- conflict_stream_scalar +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql index ecd6deee4576..19a85c1c7d63 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -10,7 +10,11 @@ select json_extract(table_alias._airbyte_data, "$['partition']") as `partition`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_complex_columns_resulting_into_long_names; +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql similarity index 75% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql index ed49b9abf584..a7d06c7427b8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -14,7 +14,11 @@ select cast(`partition` as string ) as `partition`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` --- nested_stream_with_complex_columns_resulting_into_long_names; +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql index ffc273b03241..a6fa2d3eba18 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql @@ -8,9 +8,12 @@ select _airbyte_partition_hashid, json_extract_scalar(DATA, "$['currency']") as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA cross join unnest(DATA) as DATA -where DATA is not null --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA; +where 1 = 1 +and DATA is not null; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql index 31ee702ba3ec..1e4b82a39dec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql @@ -9,7 +9,10 @@ select cast(currency as string ) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA; +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql similarity index 94% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql index 4a44b93291e8..c60a413f2d05 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_DATA_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` tmp --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA; +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql index fae58456c350..556ff2471ce9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -9,8 +9,11 @@ select json_extract_array(`partition`, "$['double_array_data']") as double_array_data, json_extract_array(`partition`, "$['DATA']") as DATA, json_extract_array(`partition`, "$['column___with__quotes']") as column___with__quotes, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` as table_alias -where `partition` is not null --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition; +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql index a0eeb3ef422b..0dbb6ecbb1f9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql @@ -9,7 +9,10 @@ select double_array_data, DATA, column___with__quotes, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition; +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql index 99eba178cf0c..56fd4a34bd7f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql @@ -18,5 +18,6 @@ select ))) as _airbyte_partition_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` tmp --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition; +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql index 1e59ead141ba..e7ab56070330 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql @@ -8,9 +8,12 @@ select _airbyte_partition_hashid, json_extract_scalar(column___with__quotes, "$['currency']") as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes cross join unnest(column___with__quotes) as column___with__quotes -where column___with__quotes is not null --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes; +where 1 = 1 +and column___with__quotes is not null; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql index be9eef8648af..b552081271d7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql @@ -9,7 +9,10 @@ select cast(currency as string ) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes; +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql index b76ca1ee8599..cdc26222cfab 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_column___with__quotes_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` tmp --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes; +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql index bc13ba7f2783..18469ff3ada9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -8,9 +8,12 @@ select _airbyte_partition_hashid, json_extract_scalar(double_array_data, "$['id']") as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data cross join unnest(double_array_data) as double_array_data -where double_array_data is not null --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data; +where 1 = 1 +and double_array_data is not null; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql index 8de95f382eb3..bc3564b761f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql @@ -9,7 +9,10 @@ select cast(id as string ) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data; +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql index 19fca86225bc..410d26ac7a07 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_double_array_data_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` tmp --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data; +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql index d00f9c167c3a..eae11107c950 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql @@ -7,7 +7,10 @@ select json_extract_scalar(_airbyte_data, "$['id']") as id, json_extract_scalar(_airbyte_data, "$['date']") as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias --- non_nested_stream_without_namespace_resulting_into_long_names; +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql index e653ba297d30..3006d17a73e9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql @@ -11,7 +11,10 @@ select cast(date as string ) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` --- non_nested_stream_without_namespace_resulting_into_long_names; +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql index 7b33b70adccb..707db214afcd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` tmp --- non_nested_stream_without_namespace_resulting_into_long_names; +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 74d16273b1c1..043cd72e489d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -7,7 +7,11 @@ select json_extract_scalar(_airbyte_data, "$['id']") as id, json_extract_array(_airbyte_data, "$['children']") as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias as table_alias --- unnest_alias; +-- unnest_alias +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql similarity index 74% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index ff4c7a287bc3..e99a33441a66 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -9,7 +9,11 @@ select int64 ) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` --- unnest_alias; +-- unnest_alias +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql similarity index 94% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 7adcbf0136c8..d807cce078ad 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -14,5 +14,7 @@ select ))) as _airbyte_unnest_alias_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` tmp --- unnest_alias; +-- unnest_alias +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 0dd96b131801..9896f675369f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -11,9 +11,12 @@ select json_extract(children, "$['owner']") as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization.`unnest_alias` as table_alias +-- children at unnest_alias/children cross join unnest(children) as children -where children is not null --- children at unnest_alias/children; +where 1 = 1 +and children is not null; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql similarity index 75% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index fc350690e2d4..4347ac1d1e98 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -12,7 +12,10 @@ select cast(owner as string ) as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` --- children at unnest_alias/children; +-- children at unnest_alias/children +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index db47bd2eac1e..07b1fea81aa2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -16,5 +16,6 @@ select ))) as _airbyte_children_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` tmp --- children at unnest_alias/children; +-- children at unnest_alias/children +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 7e66ffa11d0e..0af940c468cf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -7,8 +7,11 @@ select _airbyte_children_hashid, json_extract_scalar(owner, "$['owner_id']") as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization.`unnest_alias_children` as table_alias -where owner is not null --- owner at unnest_alias/children/owner; +-- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index 58a22b1b80d2..1d59670f9820 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -9,7 +9,10 @@ select cast(owner_id as int64 ) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` --- owner at unnest_alias/children/owner; +-- owner at unnest_alias/children/owner +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql similarity index 90% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index 2e2fffb17d68..4c1f50bc18a4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_owner_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp --- owner at unnest_alias/children/owner; +-- owner at unnest_alias/children/owner +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql similarity index 77% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql index c7e36ed1112d..7d71f4199706 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql @@ -7,7 +7,10 @@ select json_extract_scalar(_airbyte_data, "$['id']") as id, json_extract_scalar(_airbyte_data, "$['date']") as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias --- simple_stream_with_namespace_resulting_into_long_names; +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql similarity index 74% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql index d7cebc4de520..51c4df2f7fba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql @@ -11,7 +11,10 @@ select cast(date as string ) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` --- simple_stream_with_namespace_resulting_into_long_names; +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql similarity index 89% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql index 2b005f0f853c..b86eed32466f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql @@ -14,5 +14,6 @@ select ))) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` tmp --- simple_stream_with_namespace_resulting_into_long_names; +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..11c5f0e7d7c9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,86 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` + partition by range_bucket( + _airbyte_active_row, + generate_array(0, 1, 1) + ) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +with + +input_data as ( + select * + from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab3` + -- nested_stream_with_complex_columns_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_unique_key, + id, + date, + `partition`, + date as _airbyte_start_at, + lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as + string +), ''), '-', coalesce(cast(_airbyte_start_at as + string +), ''), '-', coalesce(cast(_airbyte_emitted_at as + string +), '')) as + string +))) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + date, + `partition`, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql similarity index 72% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql index 721d03c1e503..17d17751b157 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -1,20 +1,25 @@ create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at OPTIONS() as ( -- Final base SQL model select + _airbyte_unique_key, id, date, `partition`, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` -- nested_stream_with_complex_columns_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 + ); - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql similarity index 70% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql index 8b0cc5c7f1a1..a3e70d30888d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,13 @@ select id, children, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_unnest_alias_hashid from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab3` -- unnest_alias from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + ); - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql index efb1c7223ca9..f641bb360a1b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_array` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` -- conflict_stream_array from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql index c660adb89754..2a7669945831 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` -- conflict_stream_name from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql similarity index 77% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 1083f66f46a0..1d897d05e549 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` -- conflict_stream_name at conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name` +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql index 573cfc4a54fb..033f337977b9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select _airbyte_conflict_stream_name_2_hashid, `groups`, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 40652d62ec2c..b5b99cd91982 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_scalar` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` -- conflict_stream_scalar from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index c5cfbbd0b526..9e5aca960cfd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -12,9 +12,12 @@ select double_array_data, DATA, column___with__quotes, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_partition_hashid from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql index 42cacd429408..d8f8943f5149 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_DATA_hashid from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql index ef3a1635a6e4..01a20f04da09 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index 088994fc6fc1..639fa690b35b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_double_array_data_hashid from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql index 271d231a59f9..880063feec3c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` -- non_nested_stream_without_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql index e40743c250bf..b0aa1b41329f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -11,9 +11,12 @@ select _airbyte_unnest_alias_hashid, ab_id, owner, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_children_hashid from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` -- children at unnest_alias/children from `dataline-integration-testing`.test_normalization.`unnest_alias` +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 7bc307d7269f..4be0121256bd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children_owner` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_owner_hashid from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` -- owner at unnest_alias/children/owner from `dataline-integration-testing`.test_normalization.`unnest_alias_children` +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql index cf45f438b2ec..b2236adf2322 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -10,9 +10,12 @@ select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` -- simple_stream_with_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql similarity index 89% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql index cdf282290087..36e9afe3d413 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -16,5 +16,7 @@ select ))) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` tmp --- nested_stream_with_complex_columns_resulting_into_long_names; +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index f724978e5d33..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index d3df56ab3d35..17c06b46b9de 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2f0da1bb06e7..b9f5ca00d5f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_array_ab1') }} -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 0a92c655f92a..925451397e03 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('conflict_stream_array_ab2') }} tmp -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 8556d7f50afb..850e03194fc5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index a1d77f379bce..1d6d46e0d1ff 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_ab1') }} -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index dc562af89ba1..1ea9d02a7136 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('conflict_stream_name_ab2') }} tmp -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index d6e2edb75ced..e6c77c8bc197 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_hashid, {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 463a878933f8..80bebf6d45b6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index f8d96123ce25..eaec60ab1634 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql index e29eff6b3c01..917c7f8d18ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_2_hashid, {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as {{ adapter.quote('groups') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql index 759e3a18d132..f20eec57368b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast({{ adapter.quote('groups') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('groups') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql index e05638772e8d..67fab3b94f9e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index be57970e399e..eb1e66fa8ee3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index cf0d9deb51cd..b66d0d244bec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_scalar_ab1') }} -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 781c4bbfc391..18e6e00c4681 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('conflict_stream_scalar_ab2') }} tmp -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql index aaa4eb63f243..2f9195fcd8f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -1,10 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql index 63bd774a716f..d8f527d720d4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -1,10 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab1') }} -- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql deleted file mode 100644 index e2c9a1304fde..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} tmp --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql index 055fa0fd6186..1e77394f833f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql @@ -1,12 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'DATA') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('DATA'), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'DATA') }} -where DATA is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('partition', 'DATA') }} +where 1 = 1 +and DATA is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql index be7b0d9e999d..d069f1a86923 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql index 8141858e0533..856be375a7cc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql index dbe34c46db5e..b5cd1da52bd6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -1,12 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as DATA, {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as column___with__quotes, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -where {{ adapter.quote('partition') }} is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and {{ adapter.quote('partition') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql index 7c2087a397f4..23d645909627 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql @@ -1,11 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, double_array_data, DATA, column___with__quotes, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql index 101b256b8bb5..c77aba11220e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +16,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql index 4202a1cef1c6..4cbc027f3f71 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql @@ -1,12 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'column___with__quotes') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('column___with__quotes'), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'column___with__quotes') }} -where column___with__quotes is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('partition', 'column___with__quotes') }} +where 1 = 1 +and column___with__quotes is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql index 076b4bfc975c..8a107f04e7e8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql index d4d5028ad31d..59b1e25f08fb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2') }} tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql index 48b7ea236ae1..e7e798b05822 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -1,12 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'double_array_data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'double_array_data') }} -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('partition', 'double_array_data') }} +where 1 = 1 +and double_array_data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql index a3e29e92dfb7..034a47f9c894 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql index 13295b47f97f..60a4c34970ac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql index fd2fbdd54863..ed2d3cd69d1d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql index e4ed8d075d22..671850765c83 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab1') }} -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql index cd80649af632..15f30e6226cf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab2') }} tmp -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 15b34c1ca7d4..5a136fe032bb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,9 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index a416538b5422..afc783083e6c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,9 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_ab1') }} -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 0c6703933b10..ce3a0718103d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,6 @@ select tmp.* from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 57abeb23159f..995c1a2bdb48 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,13 +1,22 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} select _airbyte_unnest_alias_hashid, {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias') }} as table_alias -{{ cross_join_unnest('unnest_alias', 'children') }} -where children is not null -- children at unnest_alias/children +{{ cross_join_unnest('unnest_alias', 'children') }} +where 1 = 1 +and children is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 8db853c1d601..0cb0d82d585c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_unnest_alias_hashid, cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, cast(owner as {{ type_json() }}) as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_ab1') }} -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index 9d84f3be6aa0..5110b59063a1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +15,5 @@ select tmp.* from {{ ref('unnest_alias_children_ab2') }} tmp -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 4f28c4f8b5cb..edda3677624d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_children_hashid, {{ json_extract_scalar('owner', ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children') }} as table_alias -where owner is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index a0f474c62db7..5bc8a3f96b2c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_owner_ab1') }} -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index c5f6562c34c2..16753dc228c5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql index e3ec7ffd2f82..f29ea4161b7d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql index c437ca698089..11e2b2ec37bd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab1') }} -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql index 727eabab6ac2..79e6653964c0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +14,5 @@ select tmp.* from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab2') }} tmp -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..b2389c8fd05d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,109 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + date, + {{ adapter.quote('partition') }}, + date as _airbyte_start_at, + lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + date, + {{ adapter.quote('partition') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql new file mode 100644 index 000000000000..308499910d87 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -0,0 +1,23 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + date, + {{ adapter.quote('partition') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} +-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..1c38bf472728 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,20 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index 9ffda6086e35..fd790b1b5f1a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from {{ ref('conflict_stream_array_ab3') }} -- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 8a83ccc01e31..def7986d8580 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from {{ ref('conflict_stream_name_ab3') }} -- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 94a9dc3d0139..539ff778ebdf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql index 36c23b81b82d..1f43352fa617 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, {{ adapter.quote('groups') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 9d7acc846fd0..a013b69e57aa 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from {{ ref('conflict_stream_scalar_ab3') }} -- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql deleted file mode 100644 index 45aafa818f6f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ /dev/null @@ -1,12 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - date, - {{ adapter.quote('partition') }}, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index 8461ffb2cabd..ef861e9fa13e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -1,12 +1,21 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, double_array_data, DATA, column___with__quotes, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_partition_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql index 4e7bf4841bd5..7b461f4989e8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_DATA_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql index 2bd228318b0d..2bcc0ec32e51 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index 50b39ee05b38..353362155623 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_double_array_data_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql deleted file mode 100644 index 06311da05fc6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - date, - {{ adapter.quote('partition') }}, - date as _airbyte_start_at, - lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab4') }} --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql index 1004a9e2f4a4..14c598a1568d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab3') }} -- non_nested_stream_without_namespace_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 7a570c3836cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 77b2b0ad854e..7e78f023a44e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,11 +1,20 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, owner, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_children_hashid from {{ ref('unnest_alias_children_ab3') }} -- children at unnest_alias/children from {{ ref('unnest_alias') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 0ccf8e33d8d6..c81e06c620b1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_owner_hashid from {{ ref('unnest_alias_children_owner_ab3') }} -- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql index ce74645f7864..54dec00c79ee 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -1,10 +1,19 @@ -{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab3') }} -- simple_stream_with_namespace_resulting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql similarity index 53% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql index 23ffaa321bd8..f4835b370e3c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +15,6 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab2') }} tmp -- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..877a01b4c107 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_array(_airbyte_data, "$['conflict_stream_array']") as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..13acc814aa35 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab1` +-- conflict_stream_array +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..b0a10f06f6fe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(array_to_string(conflict_stream_array, "|", "") as + string +), '')) as + string +))) as _airbyte_conflict_stream_array_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab2` tmp +-- conflict_stream_array +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..e32dbb75a795 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + + json_extract(table_alias._airbyte_data, "$['conflict_stream_name']") + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..1e6066d334e2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(conflict_stream_name as + string +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab1` +-- conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..f8867d6b5713 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(conflict_stream_name as + string +), '')) as + string +))) as _airbyte_conflict_stream_name_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab2` tmp +-- conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..57cef056c421 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + json_extract(table_alias.conflict_stream_name, "$['conflict_stream_name']") + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..eae8466d7725 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as + string +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..0ea122e2ea01 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as + string +), ''), '-', coalesce(cast(conflict_stream_name as + string +), '')) as + string +))) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..962511141a3b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,17 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + json_extract_scalar(conflict_stream_name, "$['groups']") as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..be894b9a62a3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(`groups` as + string +) as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..c468c76f6378 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + string +), ''), '-', coalesce(cast(`groups` as + string +), '')) as + string +))) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..c933d2c41fac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['conflict_stream_scalar']") as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..f1c973037920 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(conflict_stream_scalar as + int64 +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab1` +-- conflict_stream_scalar +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..2679d90727ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(conflict_stream_scalar as + string +), '')) as + string +))) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp +-- conflict_stream_scalar +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql new file mode 100644 index 000000000000..19a85c1c7d63 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + + json_extract(table_alias._airbyte_data, "$['partition']") + as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql new file mode 100644 index 000000000000..a7d06c7427b8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -0,0 +1,24 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + cast(`partition` as + string +) as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab1` +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql new file mode 100644 index 000000000000..a6fa2d3eba18 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(DATA, "$['currency']") as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join unnest(DATA) as DATA +where 1 = 1 +and DATA is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql new file mode 100644 index 000000000000..1e4b82a39dec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + string +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab1` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql new file mode 100644 index 000000000000..c60a413f2d05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(currency as + string +), '')) as + string +))) as _airbyte_DATA_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab2` tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql new file mode 100644 index 000000000000..556ff2471ce9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + json_extract_array(`partition`, "$['double_array_data']") as double_array_data, + json_extract_array(`partition`, "$['DATA']") as DATA, + json_extract_array(`partition`, "$['column___with__quotes']") as column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql new file mode 100644 index 000000000000..0dbb6ecbb1f9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + DATA, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql new file mode 100644 index 000000000000..56fd4a34bd7f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as + string +), ''), '-', coalesce(cast(array_to_string(double_array_data, "|", "") as + string +), ''), '-', coalesce(cast(array_to_string(DATA, "|", "") as + string +), ''), '-', coalesce(cast(array_to_string(column___with__quotes, "|", "") as + string +), '')) as + string +))) as _airbyte_partition_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2` tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql new file mode 100644 index 000000000000..e7ab56070330 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(column___with__quotes, "$['currency']") as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +cross join unnest(column___with__quotes) as column___with__quotes +where 1 = 1 +and column___with__quotes is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql new file mode 100644 index 000000000000..b552081271d7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + string +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql new file mode 100644 index 000000000000..cdc26222cfab --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(currency as + string +), '')) as + string +))) as _airbyte_column___with__quotes_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2` tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql new file mode 100644 index 000000000000..18469ff3ada9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_extract_scalar(double_array_data, "$['id']") as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join unnest(double_array_data) as double_array_data +where 1 = 1 +and double_array_data is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql new file mode 100644 index 000000000000..bc3564b761f1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as + string +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql new file mode 100644 index 000000000000..410d26ac7a07 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as + string +), ''), '-', coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_double_array_data_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2` tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql new file mode 100644 index 000000000000..eae11107c950 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql new file mode 100644 index 000000000000..3006d17a73e9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab1` +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..707db214afcd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(date as + string +), '')) as + string +))) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab2` tmp +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..043cd72e489d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,17 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_array(_airbyte_data, "$['children']") as children, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..e99a33441a66 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab1` +-- unnest_alias +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..d807cce078ad --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(array_to_string(children, "|", "") as + string +), '')) as + string +))) as _airbyte_unnest_alias_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_ab2` tmp +-- unnest_alias +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..9896f675369f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,22 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + json_extract_scalar(children, "$['ab_id']") as ab_id, + + json_extract(children, "$['owner']") + as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`unnest_alias` as table_alias +-- children at unnest_alias/children +cross join unnest(children) as children +where 1 = 1 +and children is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..4347ac1d1e98 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,21 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + int64 +) as ab_id, + cast(owner as + string +) as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab1` +-- children at unnest_alias/children +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..07b1fea81aa2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,21 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as + string +), ''), '-', coalesce(cast(ab_id as + string +), ''), '-', coalesce(cast(owner as + string +), '')) as + string +))) as _airbyte_children_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab2` tmp +-- children at unnest_alias/children +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..0af940c468cf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,17 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + json_extract_scalar(owner, "$['owner_id']") as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization.`unnest_alias_children` as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..1d59670f9820 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + int64 +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab1` +-- owner at unnest_alias/children/owner +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..4c1f50bc18a4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(_airbyte_children_hashid as + string +), ''), '-', coalesce(cast(owner_id as + string +), '')) as + string +))) as _airbyte_owner_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp +-- owner at unnest_alias/children/owner +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql new file mode 100644 index 000000000000..7d71f4199706 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['date']") as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql new file mode 100644 index 000000000000..51c4df2f7fba --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + string +) as id, + cast(date as + string +) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab1` +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..b86eed32466f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(date as + string +), '')) as + string +))) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab2` tmp +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..e2187e231d38 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_scd__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`date` = DBT_INTERNAL_SOURCE.`date`,`partition` = DBT_INTERNAL_SOURCE.`partition`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `date`, `partition`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`) + values + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `date`, `partition`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql new file mode 100644 index 000000000000..c2f7397d2c3b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`date` = DBT_INTERNAL_SOURCE.`date`,`partition` = DBT_INTERNAL_SOURCE.`partition`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `date`, `partition`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`) + values + (`_airbyte_unique_key`, `id`, `date`, `partition`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..683fbd323fdb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`unnest_alias` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`unnest_alias__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_ab_id = DBT_INTERNAL_DEST._airbyte_ab_id + + + + when matched then update set + `id` = DBT_INTERNAL_SOURCE.`id`,`children` = DBT_INTERNAL_SOURCE.`children`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_unnest_alias_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_unnest_alias_hashid` + + + when not matched then insert + (`id`, `children`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_unnest_alias_hashid`) + values + (`id`, `children`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_unnest_alias_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..f641bb360a1b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_array` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_array_ab3` +-- conflict_stream_array from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..2a7669945831 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_ab3` +-- conflict_stream_name from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..1d897d05e549 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..033f337977b9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from `dataline-integration-testing`.test_normalization.`conflict_stream_name_conflict_stream_name` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..b5b99cd91982 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`conflict_stream_scalar` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`conflict_stream_scalar_ab3` +-- conflict_stream_scalar from `dataline-integration-testing`.test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..9e5aca960cfd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,23 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + DATA, + column___with__quotes, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_partition_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql new file mode 100644 index 000000000000..d8f8943f5149 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_DATA_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA_ab3` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql new file mode 100644 index 000000000000..01a20f04da09 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..639fa690b35b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from `dataline-integration-testing`.test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_partition` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..880063feec3c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`non_nested_stream_without_namespace_resulting_into_long_names_ab3` +-- non_nested_stream_without_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..b0aa1b41329f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,22 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_children_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_ab3` +-- children at unnest_alias/children from `dataline-integration-testing`.test_normalization.`unnest_alias` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..4be0121256bd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`unnest_alias_children_owner` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_owner_hashid +from `dataline-integration-testing`._airbyte_test_normalization.`unnest_alias_children_owner_ab3` +-- owner at unnest_alias/children/owner from `dataline-integration-testing`.test_normalization.`unnest_alias_children` +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..b2236adf2322 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -0,0 +1,21 @@ + + + create or replace table `dataline-integration-testing`.test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid +from `dataline-integration-testing`._airbyte_test_normalization_namespace.`simple_stream_with_namespace_resulting_into_long_names_ab3` +-- simple_stream_with_namespace_resulting_into_long_names from `dataline-integration-testing`.test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..36e9afe3d413 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -0,0 +1,22 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(`partition` as + string +), '')) as + string +))) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`nested_stream_with_complex_columns_resulting_into_long_names_ab2` tmp +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index a6e50ca2981a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab4` - OPTIONS() - as --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` tmp --- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index 683c9be94e24..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab4` - OPTIONS() - as --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` tmp --- dedup_exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 94c22e794dd8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab4` - OPTIONS() - as --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` tmp --- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx; - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 307d80f772dc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,32 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` - - - OPTIONS() - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - column___with__quotes, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab4` --- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index d164be902d73..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,38 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`dedup_exchange_rate_scd` - - - OPTIONS() - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - date as _airbyte_start_at, - lag(date) over ( - partition by id, currency, cast(NZD as - string -) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id, currency, cast(NZD as - string -) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab4` --- dedup_exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 12a8c74c9c08..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,32 +0,0 @@ - - - create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` - - - OPTIONS() - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab4` --- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index ad44e5406be9..eecd7c33da6d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -7,11 +7,14 @@ select json_extract_scalar(_airbyte_data, "$['id']") as id, json_extract_scalar(_airbyte_data, "$['name']") as name, - json_extract_scalar(_airbyte_data, "$['column___with__quotes']") as column___with__quotes, json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded; +-- dedup_cdc_excluded +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index 623a8fc37b01..995249ea89b1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -11,9 +11,6 @@ select cast(name as string ) as name, - cast(column___with__quotes as - string -) as column___with__quotes, cast(_ab_cdc_lsn as float64 ) as _ab_cdc_lsn, @@ -23,7 +20,11 @@ select cast(_ab_cdc_deleted_at as float64 ) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` --- dedup_cdc_excluded; +-- dedup_cdc_excluded +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 75554c2c95f6..aba44bffd591 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -13,7 +13,11 @@ select json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, json_extract_scalar(_airbyte_data, "$['USD']") as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate; +-- dedup_exchange_rate +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index dc63d566b553..37dc10cde3ee 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -29,7 +29,11 @@ select cast(USD as float64 ) as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` --- dedup_exchange_rate; +-- dedup_exchange_rate +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index a7ed234497d7..7e55806a2d74 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -13,7 +13,11 @@ select json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, json_extract_scalar(_airbyte_data, "$['USD']") as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias --- exchange_rate; +-- exchange_rate +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index b5299bf030b2..2bfffd7d4b36 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -29,7 +29,11 @@ select cast(USD as float64 ) as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` --- exchange_rate; +-- exchange_rate +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index cefe4b9e1dfc..4c66cad39bd6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -26,5 +26,7 @@ select ))) as _airbyte_exchange_rate_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` tmp --- exchange_rate; +-- exchange_rate +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql similarity index 86% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 90058783d998..267f6e648459 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -11,7 +11,10 @@ select json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, json_extract_scalar(_airbyte_data, "$['_ab_cdc_log_pos']") as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias --- pos_dedup_cdcx; +-- pos_dedup_cdcx +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index 176afed37e9f..58262d8eaaa5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -23,7 +23,10 @@ select cast(_ab_cdc_log_pos as float64 ) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` --- pos_dedup_cdcx; +-- pos_dedup_cdcx +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..9c881dde8e67 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..b427d44957c8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..4b61673a2f50 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,98 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` + partition by range_bucket( + _airbyte_active_row, + generate_array(0, 1, 1) + ) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +with + +input_data as ( + select * + from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` + -- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + string +), cast(_ab_cdc_updated_at as + string +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as + string +), ''), '-', coalesce(cast(_airbyte_start_at as + string +), ''), '-', coalesce(cast(_airbyte_emitted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), '')) as + string +))) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..12494cc91c84 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,82 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` + partition by range_bucket( + _airbyte_active_row, + generate_array(0, 1, 1) + ) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +with + +input_data as ( + select * + from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + -- renamed_dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as + string +), ''), '-', coalesce(cast(_airbyte_start_at as + string +), ''), '-', coalesce(cast(_airbyte_emitted_at as + string +), '')) as + string +))) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql similarity index 66% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index 248956c01dad..79eebde9c581 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -1,23 +1,27 @@ create or replace table `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at OPTIONS() as ( -- Final base SQL model select + _airbyte_unique_key, id, name, - column___with__quotes, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_dedup_cdc_excluded_hashid from `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` -- dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 + ); - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 68% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 32e1e0d363fa..fcfbe454ca5d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,13 +1,14 @@ create or replace table `dataline-integration-testing`.test_normalization.`dedup_exchange_rate` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at OPTIONS() as ( -- Final base SQL model select + _airbyte_unique_key, id, currency, date, @@ -16,10 +17,14 @@ select HKD_special___characters_1, NZD, USD, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from `dataline-integration-testing`.test_normalization.`dedup_exchange_rate_scd` -- dedup_exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 + ); - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql similarity index 74% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql index 364bdfec888d..c49e28422cd3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -1,8 +1,8 @@ create or replace table `dataline-integration-testing`.test_normalization.`exchange_rate` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_emitted_at OPTIONS() as ( @@ -16,9 +16,13 @@ select HKD_special___characters_1, NZD, USD, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` -- exchange_rate from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ); - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..1b67b9d0c597 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,23 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` +-- renamed_dedup_cdc_excluded from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..20c879115c65 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,104 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` + partition by range_bucket( + _airbyte_active_row, + generate_array(0, 1, 1) + ) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +with + +input_data as ( + select * + from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` + -- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + string +), cast(_ab_cdc_updated_at as + string +), cast(_ab_cdc_log_pos as + string +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as + string +), ''), '-', coalesce(cast(_airbyte_start_at as + string +), ''), '-', coalesce(cast(_airbyte_emitted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_log_pos as + string +), '')) as + string +))) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql similarity index 66% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index 62ca5f80f979..bde8bc2db495 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,23 +1,27 @@ create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx` - - + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at OPTIONS() as ( -- Final base SQL model select + _airbyte_unique_key, id, name, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` -- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql index e5edf0279062..662067add2e0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -9,8 +9,6 @@ select string ), ''), '-', coalesce(cast(name as string -), ''), '-', coalesce(cast(column___with__quotes as - string ), ''), '-', coalesce(cast(_ab_cdc_lsn as string ), ''), '-', coalesce(cast(_ab_cdc_updated_at as @@ -22,5 +20,7 @@ select ))) as _airbyte_dedup_cdc_excluded_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp --- dedup_cdc_excluded; +-- dedup_cdc_excluded +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 96c11756002a..4ed21e656fe2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -26,5 +26,7 @@ select ))) as _airbyte_dedup_exchange_rate_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp --- dedup_exchange_rate; +-- dedup_exchange_rate +where 1 = 1 +; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index 3cdea4f9f809..4936ef6153fe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -22,5 +22,6 @@ select ))) as _airbyte_pos_dedup_cdcx_hashid, tmp.* from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp --- pos_dedup_cdcx; +-- pos_dedup_cdcx +where 1 = 1; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..016b91d7bded --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 6144a13617e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index f1e32ebde640..fa3068bcaa9f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,22 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as column___with__quotes, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index 2b50d6d18d95..322f1de599d7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,22 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, cast(name as {{ dbt_utils.type_string() }}) as name, - cast(column___with__quotes as {{ dbt_utils.type_string() }}) as column___with__quotes, cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index aa34179c3c5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 2e5c76354fb8..a1925d6ed311 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +15,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as HKD_special___characters_1, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 4604bd5acb63..3790a0629706 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +15,11 @@ select cast(HKD_special___characters_1 as {{ dbt_utils.type_string() }}) as HKD_special___characters_1, cast(NZD as {{ dbt_utils.type_float() }}) as NZD, cast(USD as {{ dbt_utils.type_float() }}) as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index ef5f6e52d60e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 0cac51cb1e9c..51af17b7e5c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +15,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as HKD_special___characters_1, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 0182b4f09c20..319bc81f8575 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +15,11 @@ select cast(HKD_special___characters_1 as {{ dbt_utils.type_string() }}) as HKD_special___characters_1, cast(NZD as {{ dbt_utils.type_float() }}) as NZD, cast(USD as {{ dbt_utils.type_float() }}) as USD, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 07c673d18e8e..0c6628f131eb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +20,6 @@ select tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 1041493c6c67..b69171ca3358 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -7,7 +13,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index bccb189a9d7a..40c6e7e9aa85 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -7,7 +13,10 @@ select cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 7c897124b395..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..221619f8f03c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,18 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..c9cd56d5d915 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,18 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..69b881ca1ed9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,113 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 35175abb8ed5..aff096b2e6f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -9,7 +9,7 @@ with {% if is_incremental() %} new_data as ( -- retrieve incremental "new" data - select + select * from {{ ref('dedup_exchange_rate_ab3') }} -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} @@ -18,7 +18,7 @@ new_data as ( ), new_data_ids as ( -- build a subset of _airbyte_unique_key from rows that are new - select distinct + select distinct {{ dbt_utils.surrogate_key([ 'id', 'currency', diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..f008ba4b8401 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,105 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..14050eec8255 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,25 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from {{ ref('dedup_cdc_excluded_scd') }} +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..57d78ad43bf1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,28 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from {{ ref('dedup_exchange_rate_scd') }} +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..c579db34bb15 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,26 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + currency, + date, + timestamp_col, + HKD_special___characters, + HKD_special___characters_1, + NZD, + USD, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..75212a4d2aa8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,21 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..58db901dc577 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,115 @@ +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_active_row", "data_type": "int64", "range": {"start": 0, "end": 1, "interval": 1}}, + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 5b81f35bfd03..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - name, - column___with__quotes, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 0f91009def56..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - column___with__quotes, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index 85c70cd20c28..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 6f76314f0525..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - date as _airbyte_start_at, - lag(date) over ( - partition by id, currency, cast(NZD as {{ dbt_utils.type_string() }}) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id, currency, cast(NZD as {{ dbt_utils.type_string() }}) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index 49c0da07de8e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,16 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - currency, - date, - timestamp_col, - HKD_special___characters, - HKD_special___characters_1, - NZD, - USD, - _airbyte_emitted_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index ed02a5d04940..fe75b24654a8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,25 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + cluster_by = ["_airbyte_unique_key","_airbyte_emitted_at"], + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, name, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 81216ec938ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..46ac10d781f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,22 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'name', + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + ]) }} as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('dedup_cdc_excluded_ab2') }} tmp +-- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 52% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index abd8582a7f1a..2b0dd30ab3d7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +20,6 @@ select tmp.* from {{ ref('dedup_exchange_rate_ab2') }} tmp -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 52% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index 0fd3196cea9f..ee3dc14c336c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -1,4 +1,10 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -12,4 +18,5 @@ select tmp.* from {{ ref('pos_dedup_cdcx_ab2') }} tmp -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..905b8f4c4087 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,18 @@ +{{ config( + cluster_by = "_airbyte_emitted_at", + partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml index c085ca8ccc25..45c338b893ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _airbyte_raw_dedup_exchange_rate - name: _airbyte_raw_exchange_rate - name: _airbyte_raw_pos_dedup_cdcx + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..eecd7c33da6d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['name']") as name, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..995249ea89b1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,30 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(name as + string +) as name, + cast(_ab_cdc_lsn as + float64 +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float64 +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float64 +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab1` +-- dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..aba44bffd591 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..37dc10cde3ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,39 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(currency as + string +) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(HKD_special___characters_1 as + string +) as HKD_special___characters_1, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..7e55806a2d74 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['HKD_special___characters']") as HKD_special___characters_1, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..2bfffd7d4b36 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,39 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(currency as + string +) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(HKD_special___characters_1 as + string +) as HKD_special___characters_1, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..4c66cad39bd6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(HKD_special___characters_1 as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), '')) as + string +))) as _airbyte_exchange_rate_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` tmp +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..267f6e648459 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['name']") as name, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_log_pos']") as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..58262d8eaaa5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(name as + string +) as name, + cast(_ab_cdc_lsn as + float64 +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float64 +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float64 +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float64 +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab1` +-- pos_dedup_cdcx +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..9c881dde8e67 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,16 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..b427d44957c8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..bef8e97a0a47 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_dedup_cdc_excluded_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) + values + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..83688c333707 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) + values + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..5ad7a0737609 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`dedup_cdc_excluded__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_dedup_cdc_excluded_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) + values + (`_airbyte_unique_key`, `id`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_cdc_excluded_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..0691294c98c3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`dedup_exchange_rate` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`dedup_exchange_rate__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`currency` = DBT_INTERNAL_SOURCE.`currency`,`date` = DBT_INTERNAL_SOURCE.`date`,`timestamp_col` = DBT_INTERNAL_SOURCE.`timestamp_col`,`HKD_special___characters` = DBT_INTERNAL_SOURCE.`HKD_special___characters`,`HKD_special___characters_1` = DBT_INTERNAL_SOURCE.`HKD_special___characters_1`,`NZD` = DBT_INTERNAL_SOURCE.`NZD`,`USD` = DBT_INTERNAL_SOURCE.`USD`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_dedup_exchange_rate_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_dedup_exchange_rate_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `HKD_special___characters_1`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid`) + values + (`_airbyte_unique_key`, `id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `HKD_special___characters_1`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..cf06f296aea9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`exchange_rate` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`exchange_rate__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_ab_id = DBT_INTERNAL_DEST._airbyte_ab_id + + + + when matched then update set + `id` = DBT_INTERNAL_SOURCE.`id`,`currency` = DBT_INTERNAL_SOURCE.`currency`,`date` = DBT_INTERNAL_SOURCE.`date`,`timestamp_col` = DBT_INTERNAL_SOURCE.`timestamp_col`,`HKD_special___characters` = DBT_INTERNAL_SOURCE.`HKD_special___characters`,`HKD_special___characters_1` = DBT_INTERNAL_SOURCE.`HKD_special___characters_1`,`NZD` = DBT_INTERNAL_SOURCE.`NZD`,`USD` = DBT_INTERNAL_SOURCE.`USD`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_exchange_rate_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_exchange_rate_hashid` + + + when not matched then insert + (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `HKD_special___characters_1`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`) + values + (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `HKD_special___characters_1`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..0a8394cf9abc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) + values + (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..20c879115c65 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,104 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` + partition by range_bucket( + _airbyte_active_row, + generate_array(0, 1, 1) + ) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +with + +input_data as ( + select * + from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` + -- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + string +), cast(_ab_cdc_updated_at as + string +), cast(_ab_cdc_log_pos as + string +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + to_hex(md5(cast(concat(coalesce(cast(_airbyte_unique_key as + string +), ''), '-', coalesce(cast(_airbyte_start_at as + string +), ''), '-', coalesce(cast(_airbyte_emitted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_log_pos as + string +), '')) as + string +))) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..bde8bc2db495 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,27 @@ + + + create or replace table `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx` + partition by timestamp_trunc(_airbyte_emitted_at, day) + cluster by _airbyte_unique_key, _airbyte_emitted_at + OPTIONS() + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from `dataline-integration-testing`.test_normalization.`pos_dedup_cdcx_scd` +-- pos_dedup_cdcx from `dataline-integration-testing`.test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..662067add2e0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,26 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(name as + string +), ''), '-', coalesce(cast(_ab_cdc_lsn as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), '')) as + string +))) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp +-- dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..4ed21e656fe2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(HKD_special___characters_1 as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), '')) as + string +))) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..4936ef6153fe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,27 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(name as + string +), ''), '-', coalesce(cast(_ab_cdc_lsn as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), ''), '-', coalesce(cast(_ab_cdc_log_pos as + string +), '')) as + string +))) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp +-- pos_dedup_cdcx +where 1 = 1; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..016b91d7bded --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,18 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), '')) as + string +))) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..5efa724e88f2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['new_column']") as new_column, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..45be772bd787 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,39 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + float64 +) as id, + cast(currency as + string +) as currency, + cast(new_column as + float64 +) as new_column, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(NZD as + float64 +) as NZD, + cast(USD as + int64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab1` +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..cceb45e4340f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,23 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['currency']") as currency, + json_extract_scalar(_airbyte_data, "$['new_column']") as new_column, + json_extract_scalar(_airbyte_data, "$['date']") as date, + json_extract_scalar(_airbyte_data, "$['timestamp_col']") as timestamp_col, + json_extract_scalar(_airbyte_data, "$['HKD@spéçiäl & characters']") as HKD_special___characters, + json_extract_scalar(_airbyte_data, "$['NZD']") as NZD, + json_extract_scalar(_airbyte_data, "$['USD']") as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..840040e43543 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,39 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + float64 +) as id, + cast(currency as + string +) as currency, + cast(new_column as + float64 +) as new_column, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp +) as timestamp_col, + cast(HKD_special___characters as + float64 +) as HKD_special___characters, + cast(NZD as + float64 +) as NZD, + cast(USD as + float64 +) as USD, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab1` +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..79386be9f718 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(new_column as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), '')) as + string +))) as _airbyte_exchange_rate_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`exchange_rate_ab2` tmp +-- exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..8fc7d170fb05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,20 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` + OPTIONS() + as +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_extract_scalar(_airbyte_data, "$['id']") as id, + json_extract_scalar(_airbyte_data, "$['name']") as name, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_lsn']") as _ab_cdc_lsn, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_updated_at']") as _ab_cdc_updated_at, + json_extract_scalar(_airbyte_data, "$['_ab_cdc_deleted_at']") as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`.test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..295be3450166 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,30 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` + OPTIONS() + as +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + int64 +) as id, + cast(name as + string +) as name, + cast(_ab_cdc_lsn as + float64 +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float64 +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float64 +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + CURRENT_TIMESTAMP() as _airbyte_normalized_at +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..7b882d8b693c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key_scd = DBT_INTERNAL_DEST._airbyte_unique_key_scd + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`_airbyte_unique_key_scd` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key_scd`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_start_at` = DBT_INTERNAL_SOURCE.`_airbyte_start_at`,`_airbyte_end_at` = DBT_INTERNAL_SOURCE.`_airbyte_end_at`,`_airbyte_active_row` = DBT_INTERNAL_SOURCE.`_airbyte_active_row`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at` + + + when not matched then insert + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) + values + (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..f5f7e5ca80df --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`dedup_exchange_rate` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`dedup_exchange_rate__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`currency` = DBT_INTERNAL_SOURCE.`currency`,`date` = DBT_INTERNAL_SOURCE.`date`,`timestamp_col` = DBT_INTERNAL_SOURCE.`timestamp_col`,`HKD_special___characters` = DBT_INTERNAL_SOURCE.`HKD_special___characters`,`NZD` = DBT_INTERNAL_SOURCE.`NZD`,`USD` = DBT_INTERNAL_SOURCE.`USD`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_dedup_exchange_rate_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_dedup_exchange_rate_hashid`,`new_column` = DBT_INTERNAL_SOURCE.`new_column` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid`, `new_column`) + values + (`_airbyte_unique_key`, `id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid`, `new_column`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..b20cae81e4bf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`exchange_rate` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`exchange_rate__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_ab_id = DBT_INTERNAL_DEST._airbyte_ab_id + + + + when matched then update set + `id` = DBT_INTERNAL_SOURCE.`id`,`currency` = DBT_INTERNAL_SOURCE.`currency`,`date` = DBT_INTERNAL_SOURCE.`date`,`timestamp_col` = DBT_INTERNAL_SOURCE.`timestamp_col`,`HKD_special___characters` = DBT_INTERNAL_SOURCE.`HKD_special___characters`,`NZD` = DBT_INTERNAL_SOURCE.`NZD`,`USD` = DBT_INTERNAL_SOURCE.`USD`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_exchange_rate_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_exchange_rate_hashid`,`new_column` = DBT_INTERNAL_SOURCE.`new_column` + + + when not matched then insert + (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`, `new_column`) + values + (`id`, `currency`, `date`, `timestamp_col`, `HKD_special___characters`, `NZD`, `USD`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_exchange_rate_hashid`, `new_column`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..5f8e9254f0ed --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,27 @@ + + + + + + + + merge into `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded` as DBT_INTERNAL_DEST + using ( + select * from `dataline-integration-testing`.test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` + ) as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._airbyte_unique_key = DBT_INTERNAL_DEST._airbyte_unique_key + + + + when matched then update set + `_airbyte_unique_key` = DBT_INTERNAL_SOURCE.`_airbyte_unique_key`,`id` = DBT_INTERNAL_SOURCE.`id`,`_airbyte_ab_id` = DBT_INTERNAL_SOURCE.`_airbyte_ab_id`,`_airbyte_emitted_at` = DBT_INTERNAL_SOURCE.`_airbyte_emitted_at`,`_airbyte_normalized_at` = DBT_INTERNAL_SOURCE.`_airbyte_normalized_at`,`_airbyte_renamed_dedup_cdc_excluded_hashid` = DBT_INTERNAL_SOURCE.`_airbyte_renamed_dedup_cdc_excluded_hashid`,`name` = DBT_INTERNAL_SOURCE.`name`,`_ab_cdc_lsn` = DBT_INTERNAL_SOURCE.`_ab_cdc_lsn`,`_ab_cdc_updated_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_updated_at`,`_ab_cdc_deleted_at` = DBT_INTERNAL_SOURCE.`_ab_cdc_deleted_at` + + + when not matched then insert + (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) + values + (`_airbyte_unique_key`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_renamed_dedup_cdc_excluded_hashid`, `name`, `_ab_cdc_lsn`, `_ab_cdc_updated_at`, `_ab_cdc_deleted_at`) + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..ad91ec4869a2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,32 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(currency as + string +), ''), '-', coalesce(cast(new_column as + string +), ''), '-', coalesce(cast(date as + string +), ''), '-', coalesce(cast(timestamp_col as + string +), ''), '-', coalesce(cast(HKD_special___characters as + string +), ''), '-', coalesce(cast(NZD as + string +), ''), '-', coalesce(cast(USD as + string +), '')) as + string +))) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +-- dedup_exchange_rate +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..3478ba52a70a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,26 @@ + + + create or replace view `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + OPTIONS() + as +-- SQL model to build a hash column based on the values of this record +select + to_hex(md5(cast(concat(coalesce(cast(id as + string +), ''), '-', coalesce(cast(name as + string +), ''), '-', coalesce(cast(_ab_cdc_lsn as + string +), ''), '-', coalesce(cast(_ab_cdc_updated_at as + string +), ''), '-', coalesce(cast(_ab_cdc_deleted_at as + string +), '')) as + string +))) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from `dataline-integration-testing`._airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +; + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql deleted file mode 100644 index dd027fe5ce1f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab4__dbt_tmp" as - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_strea__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab3" tmp --- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql deleted file mode 100644 index e3f6c521c475..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ /dev/null @@ -1,66 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp"','U') is not null - begin - drop table test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" as - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - "date", - "partition", - "date" as _airbyte_start_at, - lag("date") over ( - partition by id - order by "date" desc, "date" desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag("date") over ( - partition by id - order by "date" desc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab4" --- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp') - ) - DROP index test_normalization.nested_stream_with_co__lting_into_long_names_scd__dbt_tmp.test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_cci - ON test_normalization.nested_stream_with_co__lting_into_long_names_scd__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index 7c1ad19a5730..1567f2b78797 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -5,8 +5,11 @@ USE [test_normalization]; select json_value(_airbyte_data, ''$."id"'') as id, json_query(_airbyte_data, ''$."conflict_stream_array"'') as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array as table_alias -- conflict_stream_array +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 643bff0b2550..a16d74e97ea4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -6,8 +6,11 @@ select cast(id as VARCHAR(max)) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab1" -- conflict_stream_array +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 5974c570b808..e5bde2f88eab 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -15,5 +15,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab2" tmp -- conflict_stream_array +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql index f566cefad326..711edc45508d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql @@ -5,9 +5,12 @@ USE [test_normalization]; select _airbyte_conflict_stream_name_2_hashid, json_value(conflict_stream_name, ''$."groups"'') as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql index e3fc0d6ddeb9..a8f6a33ad157 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql @@ -6,8 +6,11 @@ select _airbyte_conflict_stream_name_2_hashid, cast(groups as VARCHAR(max)) as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1" -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql index 659c21a38d23..4c510332a6ee 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql @@ -14,5 +14,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2" tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index ea26e4e2cb68..8f1fc7fe16db 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -5,8 +5,11 @@ USE [test_normalization]; select json_value(_airbyte_data, ''$."id"'') as id, json_query(_airbyte_data, ''$."conflict_stream_name"'') as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name as table_alias -- conflict_stream_name +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index c3cea18602f5..1e106f88c407 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -6,8 +6,11 @@ select cast(id as VARCHAR(max)) as id, cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab1" -- conflict_stream_name +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index ba1cd49fb57a..dd136bf9bad9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -14,5 +14,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab2" tmp -- conflict_stream_name +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index e1c3ad365c80..26744bb811ae 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -5,9 +5,12 @@ USE [test_normalization]; select _airbyte_conflict_stream_name_hashid, json_query(conflict_stream_name, ''$."conflict_stream_name"'') as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization."conflict_stream_name" as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 412eb35553b9..6a630d1e24c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -5,8 +5,11 @@ USE [test_normalization]; select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index f2e51c42800e..cbe3f02a60f7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -14,5 +14,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index 0a2f7b877432..ecce4bc0d160 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -5,8 +5,11 @@ USE [test_normalization]; select json_value(_airbyte_data, ''$."id"'') as id, json_value(_airbyte_data, ''$."conflict_stream_scalar"'') as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias -- conflict_stream_scalar +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index 0f158694bcd7..0cb7a859994b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -8,8 +8,11 @@ select cast(conflict_stream_scalar as bigint ) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab1" -- conflict_stream_scalar +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index a11b99ab288b..54f208622b20 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -14,5 +14,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp -- conflict_stream_scalar +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql index 280ca6054748..d74cd1c7389f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql @@ -7,9 +7,12 @@ select json_query("partition", ''$."double_array_data"'') as double_array_data, json_query("partition", ''$."DATA"'') as "DATA", json_query("partition", ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" as table_alias -where "partition" is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql index f36721488e1c..90c0eafb817d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql @@ -7,8 +7,11 @@ select double_array_data, "DATA", "column`_''with""_quotes", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1" -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql index a65b3a07f7d1..f898a92e06da 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql @@ -19,5 +19,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2" tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql similarity index 86% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql index b70ed12d5967..deb088fe4931 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql @@ -7,8 +7,11 @@ select _airbyte_partition_hashid, json_value( "DATA".value, ''$."currency"'') as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA CROSS APPLY ( SELECT [value] = CASE @@ -17,7 +20,7 @@ from "test_normalization".test_normalization."nested_stream_with_co___long_names END FROM OPENJSON("DATA") ) AS "DATA" -where "DATA" is not null --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 +and "DATA" is not null '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql index d6d994472897..505eb01f4d31 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql @@ -6,8 +6,11 @@ select _airbyte_partition_hashid, cast(currency as VARCHAR(max)) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1" -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql index 0c24bc75d1af..89631a486695 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql @@ -14,5 +14,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2" tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql index 5aed664a26d4..96a6f2a9eacc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql @@ -7,8 +7,11 @@ select _airbyte_partition_hashid, json_value( "column`_''with""_quotes".value, ''$."currency"'') as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes CROSS APPLY ( SELECT [value] = CASE @@ -17,7 +20,7 @@ from "test_normalization".test_normalization."nested_stream_with_co___long_names END FROM OPENJSON("column`_''with""_quotes") ) AS "column`_''with""_quotes" -where "column`_''with""_quotes" is not null --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +where 1 = 1 +and "column`_''with""_quotes" is not null '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql index de1463bf2bfa..9edfb5ade031 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql @@ -6,8 +6,11 @@ select _airbyte_partition_hashid, cast(currency as VARCHAR(max)) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1" -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql index 4980782957c7..1da3ab107876 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql @@ -14,5 +14,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2" tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql index b427df0d0c57..7e83428589ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql @@ -7,8 +7,11 @@ select _airbyte_partition_hashid, json_value( double_array_data.value, ''$."id"'') as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data CROSS APPLY ( SELECT [value] = CASE @@ -17,7 +20,7 @@ from "test_normalization".test_normalization."nested_stream_with_co___long_names END FROM OPENJSON(double_array_data) ) AS double_array_data -where double_array_data is not null --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 +and double_array_data is not null '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql index 7b1302aa5bf2..1eb3236e85d9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql @@ -6,8 +6,11 @@ select _airbyte_partition_hashid, cast(id as VARCHAR(max)) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1" -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql index 410371dfe86a..b6e73981f3fb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql @@ -14,5 +14,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2" tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql similarity index 86% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql index fdffccf99ecd..e921a062d3b7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql @@ -6,8 +6,12 @@ select json_value(_airbyte_data, ''$."id"'') as id, json_value(_airbyte_data, ''$."date"'') as "date", json_query(_airbyte_data, ''$."partition"'') as "partition", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias -- nested_stream_with_co__lting_into_long_names +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql index 3143ebbdee71..5eafd0d71a6b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql @@ -8,8 +8,12 @@ select cast("date" as VARCHAR(max)) as "date", cast("partition" as VARCHAR(max)) as "partition", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1" -- nested_stream_with_co__lting_into_long_names +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql index 8c0f6ca2ce70..c81d35772445 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql @@ -5,8 +5,11 @@ USE [test_normalization]; select json_value(_airbyte_data, ''$."id"'') as id, json_value(_airbyte_data, ''$."date"'') as "date", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql index e72c7eb4bf73..4abb31531a75 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql @@ -7,8 +7,11 @@ select VARCHAR(max)) as id, cast("date" as VARCHAR(max)) as "date", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1" -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql index f2bb2f973065..094bd24079d4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql @@ -14,5 +14,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2" tmp -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 19133cf1718b..177d8bda7d07 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -5,8 +5,12 @@ USE [test_normalization]; select json_value(_airbyte_data, ''$."id"'') as id, json_query(_airbyte_data, ''$."children"'') as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization._airbyte_raw_unnest_alias as table_alias -- unnest_alias +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index f6ffc45cd9bf..a9ef564eff36 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -7,8 +7,12 @@ select bigint ) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."unnest_alias_ab1" -- unnest_alias +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index b8e254e1c0d0..4544841f480f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -15,5 +15,7 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."unnest_alias_ab2" tmp -- unnest_alias +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 0664fd667ab0..d9f07f78b06c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -9,8 +9,11 @@ select children.value, ''$."ab_id"'') as ab_id, json_query( children.value, ''$."owner"'') as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children CROSS APPLY ( SELECT [value] = CASE @@ -19,7 +22,7 @@ from "test_normalization".test_normalization."unnest_alias" as table_alias END FROM OPENJSON(children) ) AS children -where children is not null --- children at unnest_alias/children +where 1 = 1 +and children is not null '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index ccf13bfd4f71..c8d4bf47083c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -8,8 +8,11 @@ select bigint ) as ab_id, cast(owner as VARCHAR(max)) as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab1" -- children at unnest_alias/children +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index 8114b1045216..6ad3a1c4caf8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -15,5 +15,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab2" tmp -- children at unnest_alias/children +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 6a01d606373f..7f49a4c7f4ea 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -5,9 +5,12 @@ USE [test_normalization]; select _airbyte_children_hashid, json_value(owner, ''$."owner_id"'') as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization."unnest_alias_children" as table_alias -where owner is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index 9f77ba923a2b..9cf0dca5efb6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -7,8 +7,11 @@ select cast(owner_id as bigint ) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab1" -- owner at unnest_alias/children/owner +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index fb293fff58eb..fe2cfe2e2a98 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -14,5 +14,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp -- owner at unnest_alias/children/owner +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql index acb59a7d8af0..a86efc7f2153 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql @@ -5,8 +5,11 @@ USE [test_normalization]; select json_value(_airbyte_data, ''$."id"'') as id, json_value(_airbyte_data, ''$."date"'') as "date", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias -- simple_stream_with_na__lting_into_long_names +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql index df95c421d630..077cbe05b163 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql @@ -7,8 +7,11 @@ select VARCHAR(max)) as id, cast("date" as VARCHAR(max)) as "date", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1" -- simple_stream_with_na__lting_into_long_names +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql index 3578b978cc3d..f2f9167067b8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql @@ -14,5 +14,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2" tmp -- simple_stream_with_na__lting_into_long_names +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..fae879eca8b1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -0,0 +1,125 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__lting_into_long_names_scd" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab3" + -- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + "date", + "partition", + "date" as _airbyte_start_at, + lag("date") over ( + partition by id + order by + "date" desc, + "date" desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag("date") over ( + partition by id + order by + "date" desc, + "date" desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + "date", + "partition", + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" FROM + "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__lting_into_long_names_scd_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__lting_into_long_names_scd_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__lting_into_long_names_scd') + ) + DROP index test_normalization.nested_stream_with_co__lting_into_long_names_scd.test_normalization_nested_stream_with_co__lting_into_long_names_scd_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__lting_into_long_names_scd_cci + ON test_normalization.nested_stream_with_co__lting_into_long_names_scd + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql similarity index 69% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql index dbb47b24e2ad..eaa24483e4d7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -1,57 +1,65 @@ + USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_temp_view"','V') is not null begin - drop view test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" + drop view test_normalization."nested_stream_with_co__lting_into_long_names_temp_view" end USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp"','U') is not null + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names"','U') is not null begin - drop table test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp" + drop table test_normalization."nested_stream_with_co__lting_into_long_names" end USE [test_normalization]; - EXEC('create view test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" as + EXEC('create view test_normalization."nested_stream_with_co__lting_into_long_names_temp_view" as -- Final base SQL model select + _airbyte_unique_key, id, "date", "partition", + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_nested_strea__nto_long_names_hashid from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" -- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 + '); - SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp" FROM - "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" FROM + "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_temp_view" USE [test_normalization]; - if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_temp_view"','V') is not null begin - drop view test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" + drop view test_normalization."nested_stream_with_co__lting_into_long_names_temp_view" end use [test_normalization]; if EXISTS ( SELECT * FROM - sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp_cci' - AND object_id=object_id('test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp') + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__lting_into_long_names_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__lting_into_long_names') ) - DROP index test_normalization.nested_stream_with_co__lting_into_long_names__dbt_tmp.test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp_cci - ON test_normalization.nested_stream_with_co__lting_into_long_names__dbt_tmp + DROP index test_normalization.nested_stream_with_co__lting_into_long_names.test_normalization_nested_stream_with_co__lting_into_long_names_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__lting_into_long_names_cci + ON test_normalization.nested_stream_with_co__lting_into_long_names + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql similarity index 50% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql index 314f2b44b400..8c21dfcb78df 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -1,55 +1,62 @@ + USE [test_normalization]; - if object_id ('test_normalization."unnest_alias__dbt_tmp_temp_view"','V') is not null + if object_id ('test_normalization."unnest_alias_temp_view"','V') is not null begin - drop view test_normalization."unnest_alias__dbt_tmp_temp_view" + drop view test_normalization."unnest_alias_temp_view" end USE [test_normalization]; - if object_id ('test_normalization."unnest_alias__dbt_tmp"','U') is not null + if object_id ('test_normalization."unnest_alias"','U') is not null begin - drop table test_normalization."unnest_alias__dbt_tmp" + drop table test_normalization."unnest_alias" end USE [test_normalization]; - EXEC('create view test_normalization."unnest_alias__dbt_tmp_temp_view" as + EXEC('create view test_normalization."unnest_alias_temp_view" as -- Final base SQL model select id, children, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_unnest_alias_hashid from "test_normalization"._airbyte_test_normalization."unnest_alias_ab3" -- unnest_alias from "test_normalization".test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + '); - SELECT * INTO "test_normalization".test_normalization."unnest_alias__dbt_tmp" FROM - "test_normalization".test_normalization."unnest_alias__dbt_tmp_temp_view" + SELECT * INTO "test_normalization".test_normalization."unnest_alias" FROM + "test_normalization".test_normalization."unnest_alias_temp_view" USE [test_normalization]; - if object_id ('test_normalization."unnest_alias__dbt_tmp_temp_view"','V') is not null + if object_id ('test_normalization."unnest_alias_temp_view"','V') is not null begin - drop view test_normalization."unnest_alias__dbt_tmp_temp_view" + drop view test_normalization."unnest_alias_temp_view" end use [test_normalization]; if EXISTS ( SELECT * FROM - sys.indexes WHERE name = 'test_normalization_unnest_alias__dbt_tmp_cci' - AND object_id=object_id('test_normalization_unnest_alias__dbt_tmp') + sys.indexes WHERE name = 'test_normalization_unnest_alias_cci' + AND object_id=object_id('test_normalization_unnest_alias') ) - DROP index test_normalization.unnest_alias__dbt_tmp.test_normalization_unnest_alias__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias__dbt_tmp_cci - ON test_normalization.unnest_alias__dbt_tmp + DROP index test_normalization.unnest_alias.test_normalization_unnest_alias_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_cci + ON test_normalization.unnest_alias + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql index 105849dc8988..2ffefaacaa27 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -23,10 +23,13 @@ select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab3" -- conflict_stream_array from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."conflict_stream_array__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql index 859508823935..e2c0ff6666d5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -23,10 +23,13 @@ select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab3" -- conflict_stream_name from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."conflict_stream_name__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql index 45c332535f17..cc5f9ded53f2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -23,10 +23,13 @@ select _airbyte_conflict_stream_name_2_hashid, groups, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3" -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index a1bc83fb3a97..79de98cb1242 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -23,10 +23,13 @@ select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" -- conflict_stream_name at conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name" +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql index ca7145c9cea6..62ee6566a04a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -23,10 +23,13 @@ select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab3" -- conflict_stream_scalar from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql index f8ba28359065..a1277fe7c94e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -25,10 +25,13 @@ select double_array_data, "DATA", "column`_''with""_quotes", + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_partition_hashid from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3" -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql index a2ffbe6664b6..31f9e07c8505 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -23,10 +23,13 @@ select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_data_hashid from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3" -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql index 7321ee6468d9..81ba784289fc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -23,10 +23,13 @@ select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3" -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql index a6a8e1aaa8ce..1e10857af549 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -23,10 +23,13 @@ select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_double_array_data_hashid from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3" -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql index 9508a59004b2..5facac722a59 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -23,10 +23,13 @@ select id, "date", + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_non_nested_s__nto_long_names_hashid from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3" -- non_nested_stream_wit__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql index 071aae66aeee..8c155fdfc51b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -24,10 +24,13 @@ select _airbyte_unnest_alias_hashid, ab_id, owner, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_children_hashid from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab3" -- children at unnest_alias/children from "test_normalization".test_normalization."unnest_alias" +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."unnest_alias_children__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 8178adafcd0d..e3fbbeed2073 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -23,10 +23,13 @@ select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_owner_hashid from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab3" -- owner at unnest_alias/children/owner from "test_normalization".test_normalization."unnest_alias_children" +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql index ab9761955c89..b6091947b13a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -23,10 +23,13 @@ select id, "date", + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_simple_strea__nto_long_names_hashid from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3" -- simple_stream_with_na__lting_into_long_names from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 '); SELECT * INTO "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql index 757103dd8f36..871fcf869df0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql @@ -15,5 +15,7 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2" tmp -- nested_stream_with_co__lting_into_long_names +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index a271eb33daa9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_co__lting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_co___long_names_partition') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('nested_stream_with_co___names_partition_data') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('nested_stream_with_co__ion_double_array_data') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index d3df56ab3d35..a256ac12c37a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2f0da1bb06e7..e770c48bd904 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_array_ab1') }} -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 0a92c655f92a..4c6cb6138a52 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_array_ab2') }} tmp -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql index 77e7bd31c512..aca9d0993391 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_2_hashid, {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql index ebd4c64d6b0b..e7ef46dec94e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast(groups as {{ dbt_utils.type_string() }}) as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name____conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql index 09eaffe2cf54..839344888f3b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name____conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 8556d7f50afb..9128769dd58b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index a1d77f379bce..7612525c0943 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_ab1') }} -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index dc562af89ba1..b9d505d0445a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name_ab2') }} tmp -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index d6e2edb75ced..97ea3e982cbe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_hashid, {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 463a878933f8..80799bfca4cf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index f8d96123ce25..1cc503d67e41 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index be57970e399e..6169df7dc10b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index cf0d9deb51cd..51d157d48e14 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_scalar_ab1') }} -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 781c4bbfc391..f71e7c572052 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_scalar_ab2') }} tmp -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql index 04ef717d09bf..a0784863bc52 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_nested_strea__nto_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co__lting_into_long_names') }} as table_alias -where {{ adapter.quote('partition') }} is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and {{ adapter.quote('partition') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql index 37d5abd5baff..aafc5126e7d6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql @@ -1,11 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql index c9befa862c0a..a0546907dec4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +14,5 @@ select tmp.* from {{ ref('nested_stream_with_co___long_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql index d5c4708f7e07..46f5b7c11ae2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('DATA')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('DATA')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('DATA')) }} -where {{ adapter.quote('DATA') }} is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('partition', adapter.quote('DATA')) }} +where 1 = 1 +and {{ adapter.quote('DATA') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql index bdf11a7ed223..8c35fe50450c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___names_partition_data_ab1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql index 11cdb80699c0..a841db507bae 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co___names_partition_data_ab2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql index fa16aeba6633..8cd21390af91 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('column`_\'with""_quotes')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} -where {{ adapter.quote('column`_\'with""_quotes') }} is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column`_\'with""_quotes') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql index 2f4622e3028a..151455db8db0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co__column___with__quotes_ab1') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql index 62ba655e2e45..fca643924c08 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co__column___with__quotes_ab2') }} tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql index 5b3a0601175b..a191b72022c5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', 'double_array_data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'double_array_data') }} -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('partition', 'double_array_data') }} +where 1 = 1 +and double_array_data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql index e46023b76c26..517f1b6759ba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co__ion_double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql index 7268452cad18..5eee9d197132 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co__ion_double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql index e37aca6c48d3..3b66fee00d27 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql index 68218a656935..fa82dea1c04a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co__lting_into_long_names_ab1') }} -- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql deleted file mode 100644 index ac961f2763ba..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_strea__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('nested_stream_with_co__lting_into_long_names_ab3') }} tmp --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql index 66f8e4943db1..4214cc379780 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql index ee6798c91bc0..d68cc555b13e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('non_nested_stream_wit__lting_into_long_names_ab1') }} -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql index 5bdd8ae4c448..b7f3b08618eb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('non_nested_stream_wit__lting_into_long_names_ab2') }} tmp -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 15b34c1ca7d4..a7877e1d2dde 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index a416538b5422..b397e46d7b04 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_ab1') }} -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 0c6703933b10..9f2bd6c9d3af 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,6 @@ select tmp.* from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 57abeb23159f..7d715421a2ab 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} select _airbyte_unnest_alias_hashid, {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias') }} as table_alias -{{ cross_join_unnest('unnest_alias', 'children') }} -where children is not null -- children at unnest_alias/children +{{ cross_join_unnest('unnest_alias', 'children') }} +where 1 = 1 +and children is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 8db853c1d601..cfdd1e1a68e9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_unnest_alias_hashid, cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, cast(owner as {{ type_json() }}) as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_ab1') }} -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index 9d84f3be6aa0..eb45635c9b9a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +13,5 @@ select tmp.* from {{ ref('unnest_alias_children_ab2') }} tmp -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 4f28c4f8b5cb..3c4f63bb121b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_children_hashid, {{ json_extract_scalar('owner', ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children') }} as table_alias -where owner is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index a0f474c62db7..109a7652533a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_owner_ab1') }} -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index c5f6562c34c2..53a716a401c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql index 1afa8d54d15e..52601ad2201a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql index 185b0e2f8a39..28487a3d51cd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('simple_stream_with_na__lting_into_long_names_ab1') }} -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql index 5766cc587575..0e264457735e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('simple_stream_with_na__lting_into_long_names_ab2') }} tmp -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..b9817bc561a3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -0,0 +1,107 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('nested_stream_with_co__lting_into_long_names_ab3') }} + -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('nested_stream_with_co__lting_into_long_names_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('nested_stream_with_co__lting_into_long_names_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('nested_stream_with_co__lting_into_long_names_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('nested_stream_with_co__lting_into_long_names_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('nested_stream_with_co__lting_into_long_names_ab3') }} + -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by id + order by + {{ adapter.quote('date') }} desc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by id + order by + {{ adapter.quote('date') }} desc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql similarity index 57% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql index 28635caec9e9..16bc6c540153 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -1,12 +1,21 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, {{ adapter.quote('date') }}, {{ adapter.quote('partition') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_nested_strea__nto_long_names_hashid from {{ ref('nested_stream_with_co__lting_into_long_names_scd') }} -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..dad253380255 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,18 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index 9ffda6086e35..6e616cb068a4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from {{ ref('conflict_stream_array_ab3') }} -- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 8a83ccc01e31..6b70e1601f56 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from {{ ref('conflict_stream_name_ab3') }} -- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql index bc23350b4f48..5244eb2cb066 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, groups, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from {{ ref('conflict_stream_name____conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 94a9dc3d0139..d309f67ec032 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 9d7acc846fd0..3e196df09e88 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from {{ ref('conflict_stream_scalar_ab3') }} -- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql index 463ee8735a5d..223dd426b23f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -1,12 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_partition_hashid from {{ ref('nested_stream_with_co___long_names_partition_ab3') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_co__lting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql index 9ab923f69627..e9f3fe55ca77 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_data_hashid from {{ ref('nested_stream_with_co___names_partition_data_ab3') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql index 8b126445559b..41147ab78466 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from {{ ref('nested_stream_with_co__column___with__quotes_ab3') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql index d36f1659ee95..0ef5f8b3ad14 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_double_array_data_hashid from {{ ref('nested_stream_with_co__ion_double_array_data_ab3') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql deleted file mode 100644 index c3ed2b7aad48..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by id - order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by id - order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from {{ ref('nested_stream_with_co__lting_into_long_names_ab4') }} --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql index 0cb2abc14b0e..ac7044df4e60 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_non_nested_s__nto_long_names_hashid from {{ ref('non_nested_stream_wit__lting_into_long_names_ab3') }} -- non_nested_stream_wit__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 7a570c3836cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 77b2b0ad854e..aaeaad626502 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,11 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, owner, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_children_hashid from {{ ref('unnest_alias_children_ab3') }} -- children at unnest_alias/children from {{ ref('unnest_alias') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 0ccf8e33d8d6..e6710c9eb232 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_owner_hashid from {{ ref('unnest_alias_children_owner_ab3') }} -- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql index 11a61e98ee75..a42310343b41 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_simple_strea__nto_long_names_hashid from {{ ref('simple_stream_with_na__lting_into_long_names_ab3') }} -- simple_stream_with_na__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql index 0c9056f13886..58cec3241028 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +13,6 @@ select tmp.* from {{ ref('nested_stream_with_co__lting_into_long_names_ab2') }} tmp -- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..1567f2b78797 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."conflict_stream_array"'') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..a16d74e97ea4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab1" +-- conflict_stream_array +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..e5bde2f88eab --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,20 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(conflict_stream_array as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_array_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab2" tmp +-- conflict_stream_array +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..711edc45508d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + json_value(conflict_stream_name, ''$."groups"'') as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..a8f6a33ad157 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as + VARCHAR(max)) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..4c510332a6ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(groups as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..8f1fc7fe16db --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."conflict_stream_name"'') as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..1e106f88c407 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab1" +-- conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..dd136bf9bad9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab2" tmp +-- conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..26744bb811ae --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + json_query(conflict_stream_name, ''$."conflict_stream_name"'') as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..6a630d1e24c1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..cbe3f02a60f7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..ecce4bc0d160 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."conflict_stream_scalar"'') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..0cb7a859994b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab1" +-- conflict_stream_scalar +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..54f208622b20 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_scalar as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp +-- conflict_stream_scalar +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql new file mode 100644 index 000000000000..d74cd1c7389f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + json_query("partition", ''$."double_array_data"'') as double_array_data, + json_query("partition", ''$."DATA"'') as "DATA", + json_query("partition", ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql new file mode 100644 index 000000000000..90c0eafb817d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab1" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql new file mode 100644 index 000000000000..f898a92e06da --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql @@ -0,0 +1,24 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(double_array_data as + VARCHAR(max)) as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("DATA" as + VARCHAR(max)) as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("column`_''with""_quotes" as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_partition_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab2" tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql new file mode 100644 index 000000000000..deb088fe4931 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + "DATA".value, ''$."currency"'') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON("DATA") + ) AS "DATA" +where 1 = 1 +and "DATA" is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql new file mode 100644 index 000000000000..505eb01f4d31 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + VARCHAR(max)) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab1" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql new file mode 100644 index 000000000000..89631a486695 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_data_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab2" tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql new file mode 100644 index 000000000000..96a6f2a9eacc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + "column`_''with""_quotes".value, ''$."currency"'') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON("column`_''with""_quotes") + ) AS "column`_''with""_quotes" +where 1 = 1 +and "column`_''with""_quotes" is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql new file mode 100644 index 000000000000..9edfb5ade031 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + VARCHAR(max)) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab1" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql new file mode 100644 index 000000000000..1da3ab107876 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_column___with__quotes_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab2" tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql new file mode 100644 index 000000000000..7e83428589ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + double_array_data.value, ''$."id"'') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON(double_array_data) + ) AS double_array_data +where 1 = 1 +and double_array_data is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql new file mode 100644 index 000000000000..1eb3236e85d9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as + VARCHAR(max)) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab1" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql new file mode 100644 index 000000000000..b6e73981f3fb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_double_array_data_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab2" tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..e921a062d3b7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + json_query(_airbyte_data, ''$."partition"'') as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..5eafd0d71a6b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + cast("partition" as VARCHAR(max)) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab1" +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..c81d35772445 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..4abb31531a75 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab1" +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..094bd24079d4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_non_nested_s__nto_long_names_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab2" tmp +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..177d8bda7d07 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."children"'') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..a9ef564eff36 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."unnest_alias_ab1" +-- unnest_alias +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..4544841f480f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,21 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(children as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unnest_alias_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."unnest_alias_ab2" tmp +-- unnest_alias +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..d9f07f78b06c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,28 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + json_value( + children.value, ''$."ab_id"'') as ab_id, + json_query( + children.value, ''$."owner"'') as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON(children) + ) AS children +where 1 = 1 +and children is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..c8d4bf47083c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,18 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast(owner as VARCHAR(max)) as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab1" +-- children at unnest_alias/children +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..6ad3a1c4caf8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,20 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unnest_alias_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(ab_id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(owner as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_children_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab2" tmp +-- children at unnest_alias/children +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..7f49a4c7f4ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,16 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + json_value(owner, ''$."owner_id"'') as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..9cf0dca5efb6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab1" +-- owner at unnest_alias/children/owner +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..fe2cfe2e2a98 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_children_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(owner_id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_owner_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp +-- owner at unnest_alias/children/owner +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..a86efc7f2153 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..077cbe05b163 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab1" +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..f2f9167067b8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_simple_strea__nto_long_names_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab2" tmp +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..9089838aa50f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "test_normalization".test_normalization."#nested_stream_with_co__lting_into_long_names_scd__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_strea__nto_long_names_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_strea__nto_long_names_hashid" + from "test_normalization".test_normalization."#nested_stream_with_co__lting_into_long_names_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql new file mode 100644 index 000000000000..ea2ee46a0293 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "test_normalization".test_normalization."#nested_stream_with_co__lting_into_long_names__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_strea__nto_long_names_hashid") + ( + select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_strea__nto_long_names_hashid" + from "test_normalization".test_normalization."#nested_stream_with_co__lting_into_long_names__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..cbef7f6209d1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."unnest_alias" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "test_normalization".test_normalization."#unnest_alias__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."unnest_alias" ("id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid") + ( + select "id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid" + from "test_normalization".test_normalization."#unnest_alias__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..2ffefaacaa27 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_array__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_array__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_array_ab3" +-- conflict_stream_array from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_array__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_array__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_array__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_array__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_array__dbt_tmp.test_normalization_conflict_stream_array__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_array__dbt_tmp_cci + ON test_normalization.conflict_stream_array__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..e2c0ff6666d5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_ab3" +-- conflict_stream_name from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql new file mode 100644 index 000000000000..cc5f9ded53f2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_name____conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..79de98cb1242 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..62ee6566a04a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_scalar__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from "test_normalization"._airbyte_test_normalization."conflict_stream_scalar_ab3" +-- conflict_stream_scalar from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_scalar__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_scalar__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_scalar__dbt_tmp.test_normalization_conflict_stream_scalar__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_scalar__dbt_tmp_cci + ON test_normalization.conflict_stream_scalar__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..a1277fe7c94e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,60 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_''with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_partition_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___long_names_partition_ab3" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp.test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci + ON test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..31f9e07c8505 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_data_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co___names_partition_data_ab3" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp.test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci + ON test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql new file mode 100644 index 000000000000..81ba784289fc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__column___with__quotes_ab3" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp.test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci + ON test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..1e10857af549 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__ion_double_array_data_ab3" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp.test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci + ON test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql new file mode 100644 index 000000000000..5facac722a59 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp"','U') is not null + begin + drop table test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_non_nested_s__nto_long_names_hashid +from "test_normalization"._airbyte_test_normalization."non_nested_stream_wit__lting_into_long_names_ab3" +-- non_nested_stream_wit__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" FROM + "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci' + AND object_id=object_id('test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp') + ) + DROP index test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp.test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci + ON test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..8c155fdfc51b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,59 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp"','U') is not null + begin + drop table test_normalization."unnest_alias_children__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."unnest_alias_children__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_children_hashid +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_ab3" +-- children at unnest_alias/children from "test_normalization".test_normalization."unnest_alias" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."unnest_alias_children__dbt_tmp" FROM + "test_normalization".test_normalization."unnest_alias_children__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_unnest_alias_children__dbt_tmp_cci' + AND object_id=object_id('test_normalization_unnest_alias_children__dbt_tmp') + ) + DROP index test_normalization.unnest_alias_children__dbt_tmp.test_normalization_unnest_alias_children__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children__dbt_tmp_cci + ON test_normalization.unnest_alias_children__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..e3fbbeed2073 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp"','U') is not null + begin + drop table test_normalization."unnest_alias_children_owner__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_owner_hashid +from "test_normalization"._airbyte_test_normalization."unnest_alias_children_owner_ab3" +-- owner at unnest_alias/children/owner from "test_normalization".test_normalization."unnest_alias_children" +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp" FROM + "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_unnest_alias_children_owner__dbt_tmp_cci' + AND object_id=object_id('test_normalization_unnest_alias_children_owner__dbt_tmp') + ) + DROP index test_normalization.unnest_alias_children_owner__dbt_tmp.test_normalization_unnest_alias_children_owner__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children_owner__dbt_tmp_cci + ON test_normalization.unnest_alias_children_owner__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql new file mode 100644 index 000000000000..b6091947b13a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -0,0 +1,58 @@ + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp"','U') is not null + begin + drop table test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_simple_strea__nto_long_names_hashid +from "test_normalization"._airbyte_test_normalization_namespace."simple_stream_with_na__lting_into_long_names_ab3" +-- simple_stream_with_na__lting_into_long_names from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + '); + + SELECT * INTO "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" FROM + "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci' + AND object_id=object_id('test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp') + ) + DROP index test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp.test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci + ON test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..871fcf869df0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql @@ -0,0 +1,21 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast("partition" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_nested_strea__nto_long_names_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."nested_stream_with_co__lting_into_long_names_ab2" tmp +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index 241474634fb2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab4__dbt_tmp" as - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab3" tmp --- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index 086bfc9a91d7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab4__dbt_tmp" as - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab3" tmp --- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 0ff1550ed499..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,14 +0,0 @@ -USE [test_normalization]; - execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab4__dbt_tmp" as - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab3" tmp --- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx - '); - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 1b50e2ded65d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,69 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_scd__dbt_tmp"','U') is not null - begin - drop table test_normalization."dedup_cdc_excluded_scd__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" as - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - "column`_''with""_quotes", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab4" --- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded_scd__dbt_tmp" FROM - "test_normalization".test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded_scd__dbt_tmp_cci' - AND object_id=object_id('test_normalization_dedup_cdc_excluded_scd__dbt_tmp') - ) - DROP index test_normalization.dedup_cdc_excluded_scd__dbt_tmp.test_normalization_dedup_cdc_excluded_scd__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded_scd__dbt_tmp_cci - ON test_normalization.dedup_cdc_excluded_scd__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index c6f7f77a55a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,73 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate_scd__dbt_tmp"','U') is not null - begin - drop table test_normalization."dedup_exchange_rate_scd__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" as - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - "date" as _airbyte_start_at, - lag("date") over ( - partition by id, currency, cast(nzd as - VARCHAR(max)) - order by "date" desc, "date" desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag("date") over ( - partition by id, currency, cast(nzd as - VARCHAR(max)) - order by "date" desc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab4" --- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."dedup_exchange_rate_scd__dbt_tmp" FROM - "test_normalization".test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_dedup_exchange_rate_scd__dbt_tmp_cci' - AND object_id=object_id('test_normalization_dedup_exchange_rate_scd__dbt_tmp') - ) - DROP index test_normalization.dedup_exchange_rate_scd__dbt_tmp.test_normalization_dedup_exchange_rate_scd__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_exchange_rate_scd__dbt_tmp_cci - ON test_normalization.dedup_exchange_rate_scd__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index e66831879ebe..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,69 +0,0 @@ - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - end - - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp"','U') is not null - begin - drop table test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - end - - - USE [test_normalization]; - EXEC('create view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" as - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab4" --- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_row_num = 1 - '); - - SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" FROM - "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - - - - USE [test_normalization]; - if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null - begin - drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" - end - - - use [test_normalization]; - if EXISTS ( - SELECT * FROM - sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci' - AND object_id=object_id('test_normalization_pos_dedup_cdcx_scd__dbt_tmp') - ) - DROP index test_normalization.pos_dedup_cdcx_scd__dbt_tmp.test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci - ON test_normalization.pos_dedup_cdcx_scd__dbt_tmp - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 2a936635d1f1..7ca4f64da70a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -5,12 +5,15 @@ USE [test_normalization]; select json_value(_airbyte_data, ''$."id"'') as id, json_value(_airbyte_data, ''$."name"'') as name, - json_value(_airbyte_data, ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias -- dedup_cdc_excluded +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index a289f83abc82..071d2319c025 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -8,8 +8,6 @@ select ) as id, cast(name as VARCHAR(max)) as name, - cast("column`_''with""_quotes" as - VARCHAR(max)) as "column`_''with""_quotes", cast(_ab_cdc_lsn as float ) as _ab_cdc_lsn, @@ -19,8 +17,12 @@ select cast(_ab_cdc_deleted_at as float ) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab1" -- dedup_cdc_excluded +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql similarity index 89% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 84e8fa8a4767..62877d8eede6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -11,8 +11,12 @@ select json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, json_value(_airbyte_data, ''$."NZD"'') as nzd, json_value(_airbyte_data, ''$."USD"'') as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index fc83ce6c39bc..b871d6e41b0b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -21,8 +21,12 @@ select cast(usd as float ) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab1" -- dedup_exchange_rate +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql similarity index 89% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 94dc9cd399d0..31eb5bb200ab 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -11,8 +11,12 @@ select json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, json_value(_airbyte_data, ''$."NZD"'') as nzd, json_value(_airbyte_data, ''$."USD"'') as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization._airbyte_raw_exchange_rate as table_alias -- exchange_rate +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index f24614e14702..bf6e40566af3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -21,8 +21,12 @@ select cast(usd as float ) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."exchange_rate_ab1" -- exchange_rate +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index bf8f496f0b61..13c53bf0f858 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -20,5 +20,7 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."exchange_rate_ab2" tmp -- exchange_rate +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 95187f4652a1..d210ebebcdfc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -9,8 +9,11 @@ select json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, json_value(_airbyte_data, ''$."_ab_cdc_log_pos"'') as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias -- pos_dedup_cdcx +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql similarity index 86% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index b8d7c54651e2..d23785697728 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -20,8 +20,11 @@ select cast(_ab_cdc_log_pos as float ) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab1" -- pos_dedup_cdcx +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7c05f028e95e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..362c8ff7d323 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..b98aeb069663 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,133 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_scd_temp_view"','V') is not null + begin + drop view test_normalization."dedup_cdc_excluded_scd_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_scd"','U') is not null + begin + drop table test_normalization."dedup_cdc_excluded_scd" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."dedup_cdc_excluded_scd_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab3" + -- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + VARCHAR(max)), cast(_ab_cdc_updated_at as + VARCHAR(max)) + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded_scd" FROM + "test_normalization".test_normalization."dedup_cdc_excluded_scd_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_scd_temp_view"','V') is not null + begin + drop view test_normalization."dedup_cdc_excluded_scd_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded_scd_cci' + AND object_id=object_id('test_normalization_dedup_cdc_excluded_scd') + ) + DROP index test_normalization.dedup_cdc_excluded_scd.test_normalization_dedup_cdc_excluded_scd_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded_scd_cci + ON test_normalization.dedup_cdc_excluded_scd + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..60c4f8d8eab2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,139 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_scd_temp_view"','V') is not null + begin + drop view test_normalization."dedup_exchange_rate_scd_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_scd"','U') is not null + begin + drop table test_normalization."dedup_exchange_rate_scd" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."dedup_exchange_rate_scd_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab3" + -- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + "date" as _airbyte_start_at, + lag("date") over ( + partition by id, currency, cast(nzd as + VARCHAR(max)) + order by + "date" desc, + "date" desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag("date") over ( + partition by id, currency, cast(nzd as + VARCHAR(max)) + order by + "date" desc, + "date" desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."dedup_exchange_rate_scd" FROM + "test_normalization".test_normalization."dedup_exchange_rate_scd_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_scd_temp_view"','V') is not null + begin + drop view test_normalization."dedup_exchange_rate_scd_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_dedup_exchange_rate_scd_cci' + AND object_id=object_id('test_normalization_dedup_exchange_rate_scd') + ) + DROP index test_normalization.dedup_exchange_rate_scd.test_normalization_dedup_exchange_rate_scd_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_exchange_rate_scd_cci + ON test_normalization.dedup_exchange_rate_scd + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..967080a2b6ae --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,121 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded_scd_temp_view"','V') is not null + begin + drop view test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded_scd"','U') is not null + begin + drop table test_normalization."renamed_dedup_cdc_excluded_scd" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" + -- renamed_dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" FROM + "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded_scd_temp_view"','V') is not null + begin + drop view test_normalization."renamed_dedup_cdc_excluded_scd_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_renamed_dedup_cdc_excluded_scd_cci' + AND object_id=object_id('test_normalization_renamed_dedup_cdc_excluded_scd') + ) + DROP index test_normalization.renamed_dedup_cdc_excluded_scd.test_normalization_renamed_dedup_cdc_excluded_scd_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_renamed_dedup_cdc_excluded_scd_cci + ON test_normalization.renamed_dedup_cdc_excluded_scd + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql similarity index 52% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index 7d3a90565705..c2c86219d046 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -1,60 +1,67 @@ + USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view"','V') is not null + if object_id ('test_normalization."dedup_cdc_excluded_temp_view"','V') is not null begin - drop view test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" + drop view test_normalization."dedup_cdc_excluded_temp_view" end USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded__dbt_tmp"','U') is not null + if object_id ('test_normalization."dedup_cdc_excluded"','U') is not null begin - drop table test_normalization."dedup_cdc_excluded__dbt_tmp" + drop table test_normalization."dedup_cdc_excluded" end USE [test_normalization]; - EXEC('create view test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" as + EXEC('create view test_normalization."dedup_cdc_excluded_temp_view" as -- Final base SQL model select + _airbyte_unique_key, id, name, - "column`_''with""_quotes", _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_dedup_cdc_excluded_hashid from "test_normalization".test_normalization."dedup_cdc_excluded_scd" -- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 + '); - SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded__dbt_tmp" FROM - "test_normalization".test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" + SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded" FROM + "test_normalization".test_normalization."dedup_cdc_excluded_temp_view" USE [test_normalization]; - if object_id ('test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view"','V') is not null + if object_id ('test_normalization."dedup_cdc_excluded_temp_view"','V') is not null begin - drop view test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" + drop view test_normalization."dedup_cdc_excluded_temp_view" end use [test_normalization]; if EXISTS ( SELECT * FROM - sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded__dbt_tmp_cci' - AND object_id=object_id('test_normalization_dedup_cdc_excluded__dbt_tmp') + sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded_cci' + AND object_id=object_id('test_normalization_dedup_cdc_excluded') ) - DROP index test_normalization.dedup_cdc_excluded__dbt_tmp.test_normalization_dedup_cdc_excluded__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded__dbt_tmp_cci - ON test_normalization.dedup_cdc_excluded__dbt_tmp + DROP index test_normalization.dedup_cdc_excluded.test_normalization_dedup_cdc_excluded_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded_cci + ON test_normalization.dedup_cdc_excluded + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 54% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 8d67ed36355d..e982dbeed83c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,26 +1,28 @@ + USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate__dbt_tmp_temp_view"','V') is not null + if object_id ('test_normalization."dedup_exchange_rate_temp_view"','V') is not null begin - drop view test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" + drop view test_normalization."dedup_exchange_rate_temp_view" end USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate__dbt_tmp"','U') is not null + if object_id ('test_normalization."dedup_exchange_rate"','U') is not null begin - drop table test_normalization."dedup_exchange_rate__dbt_tmp" + drop table test_normalization."dedup_exchange_rate" end USE [test_normalization]; - EXEC('create view test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" as + EXEC('create view test_normalization."dedup_exchange_rate_temp_view" as -- Final base SQL model select + _airbyte_unique_key, id, currency, "date", @@ -29,34 +31,40 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from "test_normalization".test_normalization."dedup_exchange_rate_scd" -- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 + '); - SELECT * INTO "test_normalization".test_normalization."dedup_exchange_rate__dbt_tmp" FROM - "test_normalization".test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" + SELECT * INTO "test_normalization".test_normalization."dedup_exchange_rate" FROM + "test_normalization".test_normalization."dedup_exchange_rate_temp_view" USE [test_normalization]; - if object_id ('test_normalization."dedup_exchange_rate__dbt_tmp_temp_view"','V') is not null + if object_id ('test_normalization."dedup_exchange_rate_temp_view"','V') is not null begin - drop view test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" + drop view test_normalization."dedup_exchange_rate_temp_view" end use [test_normalization]; if EXISTS ( SELECT * FROM - sys.indexes WHERE name = 'test_normalization_dedup_exchange_rate__dbt_tmp_cci' - AND object_id=object_id('test_normalization_dedup_exchange_rate__dbt_tmp') + sys.indexes WHERE name = 'test_normalization_dedup_exchange_rate_cci' + AND object_id=object_id('test_normalization_dedup_exchange_rate') ) - DROP index test_normalization.dedup_exchange_rate__dbt_tmp.test_normalization_dedup_exchange_rate__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_exchange_rate__dbt_tmp_cci - ON test_normalization.dedup_exchange_rate__dbt_tmp + DROP index test_normalization.dedup_exchange_rate.test_normalization_dedup_exchange_rate_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_exchange_rate_cci + ON test_normalization.dedup_exchange_rate + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql similarity index 53% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql index 4c6508f2840c..55dca5724089 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -1,23 +1,24 @@ + USE [test_normalization]; - if object_id ('test_normalization."exchange_rate__dbt_tmp_temp_view"','V') is not null + if object_id ('test_normalization."exchange_rate_temp_view"','V') is not null begin - drop view test_normalization."exchange_rate__dbt_tmp_temp_view" + drop view test_normalization."exchange_rate_temp_view" end USE [test_normalization]; - if object_id ('test_normalization."exchange_rate__dbt_tmp"','U') is not null + if object_id ('test_normalization."exchange_rate"','U') is not null begin - drop table test_normalization."exchange_rate__dbt_tmp" + drop table test_normalization."exchange_rate" end USE [test_normalization]; - EXEC('create view test_normalization."exchange_rate__dbt_tmp_temp_view" as + EXEC('create view test_normalization."exchange_rate_temp_view" as -- Final base SQL model select @@ -29,33 +30,39 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from "test_normalization"._airbyte_test_normalization."exchange_rate_ab3" -- exchange_rate from "test_normalization".test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + '); - SELECT * INTO "test_normalization".test_normalization."exchange_rate__dbt_tmp" FROM - "test_normalization".test_normalization."exchange_rate__dbt_tmp_temp_view" + SELECT * INTO "test_normalization".test_normalization."exchange_rate" FROM + "test_normalization".test_normalization."exchange_rate_temp_view" USE [test_normalization]; - if object_id ('test_normalization."exchange_rate__dbt_tmp_temp_view"','V') is not null + if object_id ('test_normalization."exchange_rate_temp_view"','V') is not null begin - drop view test_normalization."exchange_rate__dbt_tmp_temp_view" + drop view test_normalization."exchange_rate_temp_view" end use [test_normalization]; if EXISTS ( SELECT * FROM - sys.indexes WHERE name = 'test_normalization_exchange_rate__dbt_tmp_cci' - AND object_id=object_id('test_normalization_exchange_rate__dbt_tmp') + sys.indexes WHERE name = 'test_normalization_exchange_rate_cci' + AND object_id=object_id('test_normalization_exchange_rate') ) - DROP index test_normalization.exchange_rate__dbt_tmp.test_normalization_exchange_rate__dbt_tmp_cci - CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_exchange_rate__dbt_tmp_cci - ON test_normalization.exchange_rate__dbt_tmp + DROP index test_normalization.exchange_rate.test_normalization_exchange_rate_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_exchange_rate_cci + ON test_normalization.exchange_rate + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..8ddf455c5ec9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,63 @@ + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded_temp_view"','V') is not null + begin + drop view test_normalization."renamed_dedup_cdc_excluded_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded"','U') is not null + begin + drop table test_normalization."renamed_dedup_cdc_excluded" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."renamed_dedup_cdc_excluded_temp_view" as + +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" +-- renamed_dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + '); + + SELECT * INTO "test_normalization".test_normalization."renamed_dedup_cdc_excluded" FROM + "test_normalization".test_normalization."renamed_dedup_cdc_excluded_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."renamed_dedup_cdc_excluded_temp_view"','V') is not null + begin + drop view test_normalization."renamed_dedup_cdc_excluded_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_renamed_dedup_cdc_excluded_cci' + AND object_id=object_id('test_normalization_renamed_dedup_cdc_excluded') + ) + DROP index test_normalization.renamed_dedup_cdc_excluded.test_normalization_renamed_dedup_cdc_excluded_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_renamed_dedup_cdc_excluded_cci + ON test_normalization.renamed_dedup_cdc_excluded + + + + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..749435773423 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,134 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp"','U') is not null + begin + drop table test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + VARCHAR(max)), cast(_ab_cdc_updated_at as + VARCHAR(max)), cast(_ab_cdc_log_pos as + VARCHAR(max)) + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" FROM + "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci' + AND object_id=object_id('test_normalization_pos_dedup_cdcx_scd__dbt_tmp') + ) + DROP index test_normalization.pos_dedup_cdcx_scd__dbt_tmp.test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci + ON test_normalization.pos_dedup_cdcx_scd__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql similarity index 93% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index 0db7ca237be8..846de1108761 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -21,17 +21,21 @@ -- Final base SQL model select + _airbyte_unique_key, id, name, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from "test_normalization".test_normalization."pos_dedup_cdcx_scd" -- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 '); SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp" FROM diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql index 3dc43da1b4b0..71b4b90ce304 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -9,7 +9,6 @@ select concat(concat(coalesce(cast(id as VARCHAR(max)), ''''), ''-'', coalesce(cast(name as - VARCHAR(max)), ''''), ''-'', coalesce(cast("column`_''with""_quotes" as VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as @@ -18,5 +17,7 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp -- dedup_cdc_excluded +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 3411a2a70ae4..021c2a7b88f4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -20,5 +20,7 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp -- dedup_exchange_rate +where 1 = 1 + '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index 13903db7e117..fe243c2ddc74 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -18,5 +18,6 @@ select tmp.* from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp -- pos_dedup_cdcx +where 1 = 1 '); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..6781d88c4b0f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 6144a13617e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 71f6448f6532..bc0150c8df4e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index 2f1c85715033..37994c4823f9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, cast(name as {{ dbt_utils.type_string() }}) as name, - cast({{ adapter.quote('column`_\'with""_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column`_\'with""_quotes') }}, cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index aa34179c3c5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index c9a6865ef7fd..5ed2b660ee09 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 539db2675e74..c20798c70b01 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +13,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index ef5f6e52d60e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 3e66f6d0b4fc..c20f382cfbc0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 22521fcb655a..b737134cc6eb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +13,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 962cb57a8666..c645234a8db9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +18,6 @@ select tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 1041493c6c67..67bc11204a5c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -7,7 +11,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index bccb189a9d7a..a58511ceb6b7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -7,7 +11,10 @@ select cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 7c897124b395..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7fd3046082a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..a04f92bdcd55 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..d086aa78b7bc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,111 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..d406c4709941 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,121 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} desc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} desc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..67ab2bb4e05a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,103 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql similarity index 50% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index 2e53ecaf2bee..086676f17371 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -1,15 +1,23 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, name, - {{ adapter.quote('column`_\'with""_quotes') }}, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_cdc_excluded_hashid from {{ ref('dedup_cdc_excluded_scd') }} -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 57% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index f10a9446d7b8..06a37a760378 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,6 +1,11 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, currency, {{ adapter.quote('date') }}, @@ -9,9 +14,13 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from {{ ref('dedup_exchange_rate_scd') }} -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql similarity index 57% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql index 147399e6739d..980126a4146c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, @@ -9,8 +13,12 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from {{ ref('exchange_rate_ab3') }} -- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..8a8ff85f5902 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,19 @@ +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..4d32358f6f28 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,113 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 2e53ecaf2bee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - name, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index f6b8e8c4f3af..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 77ef36c1fe48..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index ed02a5d04940..fc341acc8a63 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,23 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, name, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index a5f9705f3642..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql similarity index 60% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql index 4e6d97a65757..dc6238af14f4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -1,10 +1,13 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ 'id', 'name', - adapter.quote('column`_\'with""_quotes'), '_ab_cdc_lsn', '_ab_cdc_updated_at', '_ab_cdc_deleted_at', @@ -12,4 +15,6 @@ select tmp.* from {{ ref('dedup_cdc_excluded_ab2') }} tmp -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 65% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 3e68860804dc..9221c1833031 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +18,6 @@ select tmp.* from {{ ref('dedup_exchange_rate_ab2') }} tmp -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 66% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index 0fd3196cea9f..7dd3908bac88 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -12,4 +16,5 @@ select tmp.* from {{ ref('pos_dedup_cdcx_ab2') }} tmp -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..f5ce3e6ba182 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml index c085ca8ccc25..45c338b893ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _airbyte_raw_dedup_exchange_rate - name: _airbyte_raw_exchange_rate - name: _airbyte_raw_pos_dedup_cdcx + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7ca4f64da70a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."name"'') as name, + json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, + json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, + json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..071d2319c025 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as + VARCHAR(max)) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab1" +-- dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..62877d8eede6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,22 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..b871d6e41b0b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,32 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse(nullif("date", '''') as date) as "date", + try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab1" +-- dedup_exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..31eb5bb200ab --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,22 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..bf6e40566af3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,32 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse(nullif("date", '''') as date) as "date", + try_parse(nullif(timestamp_col, '''') as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."exchange_rate_ab1" +-- exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..13c53bf0f858 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as + VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."exchange_rate_ab2" tmp +-- exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..d210ebebcdfc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."name"'') as name, + json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, + json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, + json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, + json_value(_airbyte_data, ''$."_ab_cdc_log_pos"'') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..d23785697728 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,30 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as + VARCHAR(max)) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab1" +-- pos_dedup_cdcx +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7c05f028e95e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,15 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..362c8ff7d323 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,17 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at +from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..a9f7cffc2289 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "test_normalization".test_normalization."#dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "test_normalization".test_normalization."#dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..6af21568463d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."dedup_exchange_rate_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "test_normalization".test_normalization."#dedup_exchange_rate_scd__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from "test_normalization".test_normalization."#dedup_exchange_rate_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..5d17dd49a167 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..0f76179fe523 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "test_normalization".test_normalization."#dedup_cdc_excluded__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."dedup_cdc_excluded" ("_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "test_normalization".test_normalization."#dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..c723e5310501 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."dedup_exchange_rate" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "test_normalization".test_normalization."#dedup_exchange_rate__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + ( + select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from "test_normalization".test_normalization."#dedup_exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..993b2a0893ae --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."exchange_rate" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "test_normalization".test_normalization."#exchange_rate__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") + ( + select "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid" + from "test_normalization".test_normalization."#exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..5b42db702fd8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "test_normalization".test_normalization."renamed_dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded__dbt_tmp" + ); + + insert into "test_normalization".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "test_normalization".test_normalization."#renamed_dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..749435773423 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,134 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp"','U') is not null + begin + drop table test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" as + +with + +input_data as ( + select * + from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + VARCHAR(max)), cast(_ab_cdc_updated_at as + VARCHAR(max)), cast(_ab_cdc_log_pos as + VARCHAR(max)) + order by _airbyte_ab_id + ) as _airbyte_row_num, + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unique_key as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_start_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_airbyte_emitted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" FROM + "test_normalization".test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx_scd__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci' + AND object_id=object_id('test_normalization_pos_dedup_cdcx_scd__dbt_tmp') + ) + DROP index test_normalization.pos_dedup_cdcx_scd__dbt_tmp.test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx_scd__dbt_tmp_cci + ON test_normalization.pos_dedup_cdcx_scd__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..846de1108761 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,64 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp"','U') is not null + begin + drop table test_normalization."pos_dedup_cdcx__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" as + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + SYSDATETIME() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from "test_normalization".test_normalization."pos_dedup_cdcx_scd" +-- pos_dedup_cdcx from "test_normalization".test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp" FROM + "test_normalization".test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."pos_dedup_cdcx__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_pos_dedup_cdcx__dbt_tmp_cci' + AND object_id=object_id('test_normalization_pos_dedup_cdcx__dbt_tmp') + ) + DROP index test_normalization.pos_dedup_cdcx__dbt_tmp.test_normalization_pos_dedup_cdcx__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_pos_dedup_cdcx__dbt_tmp_cci + ON test_normalization.pos_dedup_cdcx__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..71b4b90ce304 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,23 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(name as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp +-- dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..021c2a7b88f4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,26 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as + VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp +-- dedup_exchange_rate +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..fe243c2ddc74 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,23 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(name as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_log_pos as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp +-- pos_dedup_cdcx +where 1 = 1 + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..6781d88c4b0f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,19 @@ +USE [test_normalization]; + execute('create view _airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from "test_normalization"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + '); + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql deleted file mode 100644 index fe5092015be9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab4__dbt_tmp` as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_strea__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3` tmp --- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql deleted file mode 100644 index 88904c5291b2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ - - - create table - test_normalization.`nested_stream_with_co_1g_into_long_names_scd__dbt_tmp` - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - `date`, - `partition`, - `date` as _airbyte_start_at, - lag(`date`) over ( - partition by id - order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(`date`) over ( - partition by id - order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab4` --- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names -where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index 16064ab14c2f..b08ffaf4f715 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -7,7 +7,12 @@ select '$."id"') as id, json_extract(_airbyte_data, '$."conflict_stream_array"') as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization._airbyte_raw_conflict_stream_array as table_alias -- conflict_stream_array +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql similarity index 75% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2ab090a7e559..44b15b327edd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -5,7 +5,12 @@ select cast(id as char) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`conflict_stream_array_ab1` -- conflict_stream_array +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index c4a87eaa9ec9..857eb62be3f6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`conflict_stream_array_ab2` tmp -- conflict_stream_array +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql index e5689bca73bd..e3edc7461b2d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql @@ -8,8 +8,13 @@ select json_extract(table_alias.conflict_stream_name, '$."conflict_stream_name"') as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization.`conflict_stream_name` as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql index 7a25f59f18f0..16def64e5011 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql @@ -5,7 +5,12 @@ select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as json) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1` -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql index c77e0c72a4f7..99164479818a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2` tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql index d786dfa4a89e..94cef187f111 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql @@ -6,8 +6,13 @@ select _airbyte_conflict_stream_name_2_hashid, json_value(conflict_stream_name, '$."groups"') as `groups`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql index 3393dde64ef8..222256b4cc4e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql @@ -5,7 +5,12 @@ select _airbyte_conflict_stream_name_2_hashid, cast(`groups` as char) as `groups`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1` -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql index 24929b82dca9..2442380dac27 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2` tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index a575697fcb36..f0cefd4a6acf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -9,7 +9,12 @@ select json_extract(table_alias._airbyte_data, '$."conflict_stream_name"') as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization._airbyte_raw_conflict_stream_name as table_alias -- conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index f9ad98a47215..b4f1e6a94ea6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -5,7 +5,12 @@ select cast(id as char) as id, cast(conflict_stream_name as json) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`conflict_stream_name_ab1` -- conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index 65dd3434a822..8a7e62bbfafc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`conflict_stream_name_ab2` tmp -- conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index c97c708d399b..a30f57d7d4ed 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -7,7 +7,12 @@ select '$."id"') as id, json_value(_airbyte_data, '$."conflict_stream_scalar"') as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization._airbyte_raw_conflict_stream_scalar as table_alias -- conflict_stream_scalar +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql similarity index 77% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index fc08f53590d0..5822b0221381 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -7,7 +7,12 @@ select cast(conflict_stream_scalar as signed ) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`conflict_stream_scalar_ab1` -- conflict_stream_scalar +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 88e6caaca33d..6b31dc6f9b85 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp -- conflict_stream_scalar +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql index 81d223bc373c..2a4990a4233d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql @@ -11,7 +11,13 @@ select json_extract(table_alias._airbyte_data, '$."partition"') as `partition`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization._airbyte_raw_nested_s__lting_into_long_names as table_alias -- nested_stream_with_co__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql index 23b64ee8f8e5..a5cd56efead8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql @@ -6,7 +6,13 @@ select cast(id as char) as id, cast(`date` as char) as `date`, cast(`partition` as json) as `partition`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1` -- nested_stream_with_co__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql index 5ba25fb20ebc..328e7097da14 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql @@ -10,8 +10,13 @@ select '$."DATA"') as `DATA`, json_extract(`partition`, '$."column___with__quotes"') as `column__'with"_quotes`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization.`nested_stream_with_co__lting_into_long_names` as table_alias -where `partition` is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql index 56a4acbc626d..f5b912cd6800 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql @@ -7,7 +7,12 @@ select double_array_data, `DATA`, `column__'with"_quotes`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1` -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql index 5b0ac76d32b3..bf2e57812e3d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2` tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql index 6bb578315008..5dd3b6c8ce6f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql @@ -51,9 +51,14 @@ select _airbyte_partition_hashid, json_value(_airbyte_nested_data, '$."id"') as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql index 43a2faa29900..3280a736e4bc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql @@ -5,7 +5,12 @@ select _airbyte_partition_hashid, cast(id as char) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1` -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql index 39db1c9a56a1..f14064aa73ee 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2` tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql index a532d3e22f8f..19ae65abddfd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql @@ -51,9 +51,14 @@ select _airbyte_partition_hashid, json_value(_airbyte_nested_data, '$."currency"') as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where `DATA` is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and `DATA` is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql index 4c7968e3c11d..2a28642a67f8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql @@ -5,7 +5,12 @@ select _airbyte_partition_hashid, cast(currency as char) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1` -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql index 0bd43b99279c..14553972ba13 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2` tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql index 4f3b7632f1b1..c7861eb82bd9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql @@ -51,9 +51,14 @@ select _airbyte_partition_hashid, json_value(_airbyte_nested_data, '$."currency"') as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where `column__'with"_quotes` is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and `column__'with"_quotes` is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql index 097f8b5b835e..6c71fa4683f5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql @@ -5,7 +5,12 @@ select _airbyte_partition_hashid, cast(currency as char) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1` -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql index e9ecd923a8f4..b8a160636ce2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2` tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql index 590bb28948ca..c428b52dd85c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql @@ -7,7 +7,12 @@ select '$."id"') as id, json_value(_airbyte_data, '$."date"') as `date`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization._airbyte_raw_non_nest__lting_into_long_names as table_alias -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql index 8c37318b3d29..0957dfcb71cb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql @@ -5,7 +5,12 @@ select cast(id as char) as id, cast(`date` as char) as `date`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1` -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql index 4d0c9bc16263..9122165fc08b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2` tmp -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 6e843ca7a657..46c85dc6a760 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -7,7 +7,13 @@ select '$."id"') as id, json_extract(_airbyte_data, '$."children"') as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization._airbyte_raw_unnest_alias as table_alias -- unnest_alias +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index a7d929e77df0..d0321601fc92 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -7,7 +7,13 @@ select signed ) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`unnest_alias_ab1` -- unnest_alias +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql similarity index 96% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 4d48087571c3..9819d4b985d1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -7,4 +7,6 @@ select tmp.* from _airbyte_test_normalization.`unnest_alias_ab2` tmp -- unnest_alias +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 0b2b2a86bd2d..d781016d8076 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -55,9 +55,14 @@ select json_extract(_airbyte_nested_data, '$."owner"') as `owner`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization.`unnest_alias` as table_alias -left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid -where children is not null -- children at unnest_alias/children +left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid +where 1 = 1 +and children is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 3799355bfffe..a2e785ac473e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -8,7 +8,12 @@ select signed ) as ab_id, cast(`owner` as json) as `owner`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`unnest_alias_children_ab1` -- children at unnest_alias/children +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index e99957b0b0e5..a10d4db5544e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`unnest_alias_children_ab2` tmp -- children at unnest_alias/children +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index a8080abb06b5..942b77e12e1d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -6,8 +6,13 @@ select _airbyte_children_hashid, json_value(`owner`, '$."owner_id"') as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization.`unnest_alias_children` as table_alias -where `owner` is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and `owner` is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql similarity index 77% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index e96688a1c6cc..1f95376e9d66 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -7,7 +7,12 @@ select cast(owner_id as signed ) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`unnest_alias_children_owner_ab1` -- owner at unnest_alias/children/owner +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index ac7aeed08683..923718fab124 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp -- owner at unnest_alias/children/owner +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql index 52aa78db11bd..72c98d266114 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql @@ -7,7 +7,12 @@ select '$."id"') as id, json_value(_airbyte_data, '$."date"') as `date`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names as table_alias -- simple_stream_with_na__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql index 679f88767d21..3869d2131aac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql @@ -5,7 +5,12 @@ select cast(id as char) as id, cast(`date` as char) as `date`, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1` -- simple_stream_with_na__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql index b2c0230b903c..aa7c1c724947 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2` tmp -- simple_stream_with_na__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql new file mode 100644 index 000000000000..bfb90b5ac404 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -0,0 +1,70 @@ + + + create table + test_normalization.`nested_stream_with_co_1g_into_long_names_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3` + -- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `date`, + `partition`, + `date` as _airbyte_start_at, + lag(`date`) over ( + partition by id + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(`date`) over ( + partition by id + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `date`, + `partition`, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql similarity index 75% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql index a2c64e0181af..358de44385e3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -6,12 +6,19 @@ -- Final base SQL model select + _airbyte_unique_key, id, `date`, `partition`, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_nested_strea__nto_long_names_hashid from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` -- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql similarity index 77% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql index cb9830768576..cc143bf47b05 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -8,8 +8,14 @@ select id, children, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_unnest_alias_hashid from _airbyte_test_normalization.`unnest_alias_ab3` -- unnest_alias from test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql index 2c1eed7331a4..717bc072a956 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -8,8 +8,13 @@ select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from _airbyte_test_normalization.`conflict_stream_array_ab3` -- conflict_stream_array from test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql index d341fcab11c6..bdfe747bf5e7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -8,8 +8,13 @@ select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from _airbyte_test_normalization.`conflict_stream_name_ab3` -- conflict_stream_name from test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql index c55a5e3a62ed..ded6eda50709 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -8,8 +8,13 @@ select _airbyte_conflict_stream_name_2_hashid, `groups`, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3` -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name_conflict_stream_name` +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 8cf88cc99275..a64da6fc2bba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -8,8 +8,13 @@ select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3` -- conflict_stream_name at conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name` +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql index aad49db6aa81..b79e0e4c08a4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -8,8 +8,13 @@ select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from _airbyte_test_normalization.`conflict_stream_scalar_ab3` -- conflict_stream_scalar from test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql similarity index 86% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql index a796e4bec478..2c30ff5645d3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -10,8 +10,13 @@ select double_array_data, `DATA`, `column__'with"_quotes`, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_partition_hashid from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3` -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co__lting_into_long_names` +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql index 829800f026b8..cc91b90e22c9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -8,8 +8,13 @@ select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_data_hashid from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3` -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql index c65521b6b2c2..600adbbcf8b8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -8,8 +8,13 @@ select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3` -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql index e0795e0c729c..a584e5957c67 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -8,8 +8,13 @@ select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_double_array_data_hashid from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3` -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql index d80970404ecf..9b1d5c0c792a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -8,8 +8,13 @@ select id, `date`, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_non_nested_s__nto_long_names_hashid from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3` -- non_nested_stream_wit__lting_into_long_names from test_normalization._airbyte_raw_non_nest__lting_into_long_names +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql index 8d943344e1ea..62be462c50f8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -9,8 +9,13 @@ select _airbyte_unnest_alias_hashid, ab_id, `owner`, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_children_hashid from _airbyte_test_normalization.`unnest_alias_children_ab3` -- children at unnest_alias/children from test_normalization.`unnest_alias` +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 74ae1e9027f2..5747da2ce362 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -8,8 +8,13 @@ select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_owner_hashid from _airbyte_test_normalization.`unnest_alias_children_owner_ab3` -- owner at unnest_alias/children/owner from test_normalization.`unnest_alias_children` +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql index c7220f05c387..8a5111492bbf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -8,8 +8,13 @@ select id, `date`, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_simple_strea__nto_long_names_hashid from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3` -- simple_stream_with_na__lting_into_long_names from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names +where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql index 1b0b96e37ab7..f8c6c82e2fbc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql @@ -7,4 +7,6 @@ select tmp.* from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2` tmp -- nested_stream_with_co__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 14e4ec11e013..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_co__lting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_co___long_names_partition') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('nested_stream_with_co___names_partition_data') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('nested_stream_with_co__ion_double_array_data') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index d3df56ab3d35..a256ac12c37a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2f0da1bb06e7..e770c48bd904 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_array_ab1') }} -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 0a92c655f92a..4c6cb6138a52 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_array_ab2') }} tmp -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql index d6e2edb75ced..97ea3e982cbe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_hashid, {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql index d31ae94a8814..88e3e7b5974a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name__2flict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql index dddba298b946..b17ecef4ec82 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name__2flict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql index e29eff6b3c01..851bc06e78ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_2_hashid, {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as {{ adapter.quote('groups') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql index e131f0fc01c1..3bb11ee28a2e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast({{ adapter.quote('groups') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('groups') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name__3flict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql index 8d66a3a4b364..04a82c2387b4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name__3flict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 8556d7f50afb..9128769dd58b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index a1d77f379bce..7612525c0943 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_ab1') }} -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index dc562af89ba1..b9d505d0445a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_name_ab2') }} tmp -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index be57970e399e..6169df7dc10b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index cf0d9deb51cd..51d157d48e14 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_scalar_ab1') }} -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 781c4bbfc391..f71e7c572052 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('conflict_stream_scalar_ab2') }} tmp -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql index 8f7cabd6d907..bcca84084504 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} as table_alias -- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql index 6b268cccc087..ec9afbc12ca6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co_1g_into_long_names_ab1') }} -- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql deleted file mode 100644 index 6d2cfb296ed7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_strea__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('nested_stream_with_co_1g_into_long_names_ab3') }} tmp --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql index 31f6b6a06672..686acef42df9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_nested_strea__nto_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column__\'with"_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co__lting_into_long_names') }} as table_alias -where {{ adapter.quote('partition') }} is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and {{ adapter.quote('partition') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql index 81aae267055d..17847b387ff8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql @@ -1,11 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column__\'with"_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co_2g_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql index 579a124df198..cd19c3cd4d2a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +14,5 @@ select tmp.* from {{ ref('nested_stream_with_co_2g_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql index 5b3a0601175b..a191b72022c5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', 'double_array_data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'double_array_data') }} -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('partition', 'double_array_data') }} +where 1 = 1 +and double_array_data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql index b4357e9d1f1e..685ebd8cf4f0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co_3double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql index a8aa152099ce..d3d654a0b267 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co_3double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql index d5c4708f7e07..46f5b7c11ae2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('DATA')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('DATA')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('DATA')) }} -where {{ adapter.quote('DATA') }} is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('partition', adapter.quote('DATA')) }} +where 1 = 1 +and {{ adapter.quote('DATA') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql index baad4bbf0f14..b6912038bca1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co_3es_partition_data_ab1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql index ff7df8bea771..cc049c42b2c4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co_3es_partition_data_ab2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql index f8b1f55fc4df..8b23b31f54b9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql @@ -1,12 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('column__\'with"_quotes')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('column__\'with"_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('column__\'with"_quotes')) }} -where {{ adapter.quote('column__\'with"_quotes') }} is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('partition', adapter.quote('column__\'with"_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column__\'with"_quotes') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql index 32acb349e636..4596c0998414 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_co_3mn___with__quotes_ab1') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql index fed352d1f303..640685bc8c6a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('nested_stream_with_co_3mn___with__quotes_ab2') }} tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql index 6575f5003ddf..4e89cc3ec03c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_non_nest__lting_into_long_names') }} as table_alias -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql index d8d262e97685..c1bc0cb6750f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('non_nested_stream_wit_1g_into_long_names_ab1') }} -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql index 98a4ac30d1be..81d16c7714e9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('non_nested_stream_wit_1g_into_long_names_ab2') }} tmp -- non_nested_stream_wit__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 15b34c1ca7d4..a7877e1d2dde 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index a416538b5422..b397e46d7b04 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_ab1') }} -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 0c6703933b10..9f2bd6c9d3af 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,6 @@ select tmp.* from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 2237b274e29b..0ae376bf5c5e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} select _airbyte_unnest_alias_hashid, {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as {{ adapter.quote('owner') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias') }} as table_alias -{{ cross_join_unnest('unnest_alias', 'children') }} -where children is not null -- children at unnest_alias/children +{{ cross_join_unnest('unnest_alias', 'children') }} +where 1 = 1 +and children is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 435447589db2..4e285cc11ccc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_unnest_alias_hashid, cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, cast({{ adapter.quote('owner') }} as {{ type_json() }}) as {{ adapter.quote('owner') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_ab1') }} -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index e1fa4b470cc8..71b013d95e95 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +13,5 @@ select tmp.* from {{ ref('unnest_alias_children_ab2') }} tmp -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 9085180f4824..833b7054705a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,10 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_children_hashid, {{ json_extract_scalar(adapter.quote('owner'), ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children') }} as table_alias -where {{ adapter.quote('owner') }} is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and {{ adapter.quote('owner') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index a0f474c62db7..109a7652533a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_owner_ab1') }} -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index c5f6562c34c2..53a716a401c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql index ab36933bcef0..15a7e0bc2774 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization_namespace', '_airbyte_raw_simple_s__lting_into_long_names') }} as table_alias -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql index 0cd875e93da8..b37432da5333 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql @@ -1,9 +1,16 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('simple_stream_with_na_1g_into_long_names_ab1') }} -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql index 281544a38863..bfe41baca551 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +12,5 @@ select tmp.* from {{ ref('simple_stream_with_na_1g_into_long_names_ab2') }} tmp -- simple_stream_with_na__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql new file mode 100644 index 000000000000..98783c3a173d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -0,0 +1,107 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('nested_stream_with_co_1g_into_long_names_ab3') }} + -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('nested_stream_with_co_1g_into_long_names_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('nested_stream_with_co_1g_into_long_names_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('nested_stream_with_co_1g_into_long_names_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('nested_stream_with_co_1g_into_long_names_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('nested_stream_with_co_1g_into_long_names_ab3') }} + -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by id + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by id + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql similarity index 55% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql index a35988d6c18f..bb554d43a98d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -1,12 +1,21 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, {{ adapter.quote('date') }}, {{ adapter.quote('partition') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_nested_strea__nto_long_names_hashid from {{ ref('nested_stream_with_co_1g_into_long_names_scd') }} -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..dad253380255 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,18 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index 9ffda6086e35..6e616cb068a4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from {{ ref('conflict_stream_array_ab3') }} -- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 8a83ccc01e31..6b70e1601f56 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from {{ ref('conflict_stream_name_ab3') }} -- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql index ac40e0861b86..81f173664209 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, {{ adapter.quote('groups') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from {{ ref('conflict_stream_name__3flict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index ea20646f8121..be1b4fe446a4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from {{ ref('conflict_stream_name__2flict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 9d7acc846fd0..3e196df09e88 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from {{ ref('conflict_stream_scalar_ab3') }} -- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql deleted file mode 100644 index 71213c7a23fa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by id - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by id - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid -from {{ ref('nested_stream_with_co_1g_into_long_names_ab4') }} --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql index b64a0b856c0e..f5f0f16333f8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -1,12 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column__\'with"_quotes') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_partition_hashid from {{ ref('nested_stream_with_co_2g_names_partition_ab3') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_co__lting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql index eec219cee615..5219d4bf1eb9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_data_hashid from {{ ref('nested_stream_with_co_3es_partition_data_ab3') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql index b7c647643b54..b56e3f5b5da3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from {{ ref('nested_stream_with_co_3mn___with__quotes_ab3') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql index eb0f195e2786..1cd2763acbac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_double_array_data_hashid from {{ ref('nested_stream_with_co_3double_array_data_ab3') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_co___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql index 12fcd9b638fa..80f9bdb879a3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_non_nested_s__nto_long_names_hashid from {{ ref('non_nested_stream_wit_1g_into_long_names_ab3') }} -- non_nested_stream_wit__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nest__lting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 7a570c3836cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 9ce667c8b142..59c7f1b6c50d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,11 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, {{ adapter.quote('owner') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_children_hashid from {{ ref('unnest_alias_children_ab3') }} -- children at unnest_alias/children from {{ ref('unnest_alias') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 0ccf8e33d8d6..e6710c9eb232 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_owner_hashid from {{ ref('unnest_alias_children_owner_ab3') }} -- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql index fdd42e4ce199..da684535a739 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -1,10 +1,17 @@ -{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_simple_strea__nto_long_names_hashid from {{ ref('simple_stream_with_na_1g_into_long_names_ab3') }} -- simple_stream_with_na__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_s__lting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql index ec401f322147..1e93090d9c40 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +13,6 @@ select tmp.* from {{ ref('nested_stream_with_co_1g_into_long_names_ab2') }} tmp -- nested_stream_with_co__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..b08ffaf4f715 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_array_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_extract(_airbyte_data, + '$."conflict_stream_array"') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..44b15b327edd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_array_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_array_ab1` +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..857eb62be3f6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_array_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_array as char), '')) as char)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_array_ab2` tmp +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql new file mode 100644 index 000000000000..e3edc7461b2d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab1.sql @@ -0,0 +1,20 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + json_extract(table_alias.conflict_stream_name, + '$."conflict_stream_name"') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql new file mode 100644 index 000000000000..16def64e5011 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as json) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql new file mode 100644 index 000000000000..99164479818a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__2flict_stream_name_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql new file mode 100644 index 000000000000..94cef187f111 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + json_value(conflict_stream_name, + '$."groups"') as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`conflict_stream_name_conflict_stream_name` as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql new file mode 100644 index 000000000000..222256b4cc4e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(`groups` as char) as `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab1` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql new file mode 100644 index 000000000000..2442380dac27 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name__3flict_stream_name_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as char), ''), '-', coalesce(cast(`groups` as char), '')) as char)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab2` tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..f0cefd4a6acf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,20 @@ + + create view _airbyte_test_normalization.`conflict_stream_name_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + + json_extract(table_alias._airbyte_data, + '$."conflict_stream_name"') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..b4f1e6a94ea6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`conflict_stream_name_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(conflict_stream_name as json) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_name_ab1` +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..8a7e62bbfafc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_name_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_name as char), '')) as char)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_name_ab2` tmp +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..a30f57d7d4ed --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_scalar_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."conflict_stream_scalar"') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..5822b0221381 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`conflict_stream_scalar_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(conflict_stream_scalar as + signed +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`conflict_stream_scalar_ab1` +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..6b31dc6f9b85 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`conflict_stream_scalar_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(conflict_stream_scalar as char), '')) as char)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from _airbyte_test_normalization.`conflict_stream_scalar_ab2` tmp +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql new file mode 100644 index 000000000000..2a4990a4233d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql @@ -0,0 +1,23 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + + json_extract(table_alias._airbyte_data, + '$."partition"') + as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_nested_s__lting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql new file mode 100644 index 000000000000..a5cd56efead8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + cast(`partition` as json) as `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab1` +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql new file mode 100644 index 000000000000..328e7097da14 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql @@ -0,0 +1,22 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + json_extract(`partition`, + '$."double_array_data"') as double_array_data, + json_extract(`partition`, + '$."DATA"') as `DATA`, + json_extract(`partition`, + '$."column___with__quotes"') as `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co__lting_into_long_names` as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and `partition` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql new file mode 100644 index 000000000000..f5b912cd6800 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + `DATA`, + `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab1` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql new file mode 100644 index 000000000000..bf2e57812e3d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as char), ''), '-', coalesce(cast(double_array_data as char), ''), '-', coalesce(cast(`DATA` as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), '')) as char)) as _airbyte_partition_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab2` tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql new file mode 100644 index 000000000000..5dd3b6c8ce6f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql @@ -0,0 +1,64 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(double_array_data, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.double_array_data + where numbers.generated_number <= json_length(double_array_data) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."id"') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql new file mode 100644 index 000000000000..3280a736e4bc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as char) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab1` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql new file mode 100644 index 000000000000..f14064aa73ee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(id as char), '')) as char)) as _airbyte_double_array_data_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab2` tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql new file mode 100644 index 000000000000..19ae65abddfd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql @@ -0,0 +1,64 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(`DATA`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.`DATA` + where numbers.generated_number <= json_length(`DATA`) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."currency"') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and `DATA` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql new file mode 100644 index 000000000000..2a28642a67f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as char) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab1` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql new file mode 100644 index 000000000000..14553972ba13 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_data_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab2` tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql new file mode 100644 index 000000000000..c7861eb82bd9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab1.sql @@ -0,0 +1,64 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + + ), + joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + + json_extract(`column__'with"_quotes`, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`nested_stream_with_co___long_names_partition` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_co___long_names_partition.`column__'with"_quotes` + where numbers.generated_number <= json_length(`column__'with"_quotes`) + ) +select + _airbyte_partition_hashid, + json_value(_airbyte_nested_data, + '$."currency"') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`nested_stream_with_co___long_names_partition` as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and `column__'with"_quotes` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql new file mode 100644 index 000000000000..6c71fa4683f5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as char) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab1` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql new file mode 100644 index 000000000000..b8a160636ce2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_co_3mn___with__quotes_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_column___with__quotes_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab2` tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql new file mode 100644 index 000000000000..c428b52dd85c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_non_nest__lting_into_long_names as table_alias +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql new file mode 100644 index 000000000000..0957dfcb71cb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab1` +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql new file mode 100644 index 000000000000..9122165fc08b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wit_1g_into_long_names_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_non_nested_s__nto_long_names_hashid, + tmp.* +from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab2` tmp +-- non_nested_stream_wit__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..46c85dc6a760 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,19 @@ + + create view _airbyte_test_normalization.`unnest_alias_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_extract(_airbyte_data, + '$."children"') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..d0321601fc92 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,19 @@ + + create view _airbyte_test_normalization.`unnest_alias_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`unnest_alias_ab1` +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..9819d4b985d1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`unnest_alias_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(children as char), '')) as char)) as _airbyte_unnest_alias_hashid, + tmp.* +from _airbyte_test_normalization.`unnest_alias_ab2` tmp +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..d781016d8076 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,68 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * pow(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + + ), + joined as ( + select + _airbyte_unnest_alias_hashid as _airbyte_hashid, + + json_extract(children, concat("$[", numbers.generated_number - 1, "][0]")) as _airbyte_nested_data + from test_normalization.`unnest_alias` + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in unnest_alias.children + where numbers.generated_number <= json_length(children) + ) +select + _airbyte_unnest_alias_hashid, + json_value(_airbyte_nested_data, + '$."ab_id"') as ab_id, + + json_extract(_airbyte_nested_data, + '$."owner"') + as `owner`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`unnest_alias` as table_alias +-- children at unnest_alias/children +left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid +where 1 = 1 +and children is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..a2e785ac473e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,19 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + signed +) as ab_id, + cast(`owner` as json) as `owner`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`unnest_alias_children_ab1` +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..a10d4db5544e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_unnest_alias_hashid as char), ''), '-', coalesce(cast(ab_id as char), ''), '-', coalesce(cast(`owner` as char), '')) as char)) as _airbyte_children_hashid, + tmp.* +from _airbyte_test_normalization.`unnest_alias_children_ab2` tmp +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..942b77e12e1d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_owner_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + json_value(`owner`, + '$."owner_id"') as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization.`unnest_alias_children` as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and `owner` is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..1f95376e9d66 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_owner_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + signed +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`unnest_alias_children_owner_ab1` +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..923718fab124 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`unnest_alias_children_owner_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(_airbyte_children_hashid as char), ''), '-', coalesce(cast(owner_id as char), '')) as char)) as _airbyte_owner_hashid, + tmp.* +from _airbyte_test_normalization.`unnest_alias_children_owner_ab2` tmp +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql new file mode 100644 index 000000000000..72c98d266114 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab1.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."date"') as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names as table_alias +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql new file mode 100644 index 000000000000..3869d2131aac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab2.sql @@ -0,0 +1,16 @@ + + create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as char) as id, + cast(`date` as char) as `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab1` +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql new file mode 100644 index 000000000000..aa7c1c724947 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_na_1g_into_long_names_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), '')) as char)) as _airbyte_simple_strea__nto_long_names_hashid, + tmp.* +from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab2` tmp +-- simple_stream_with_na__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql new file mode 100644 index 000000000000..bfb90b5ac404 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -0,0 +1,70 @@ + + + create table + test_normalization.`nested_stream_with_co_1g_into_long_names_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3` + -- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `date`, + `partition`, + `date` as _airbyte_start_at, + lag(`date`) over ( + partition by id + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(`date`) over ( + partition by id + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `date`, + `partition`, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql new file mode 100644 index 000000000000..358de44385e3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -0,0 +1,24 @@ + + + create table + test_normalization.`nested_stream_with_co__lting_into_long_names__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + `date`, + `partition`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_nested_strea__nto_long_names_hashid +from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` +-- nested_stream_with_co__lting_into_long_names from test_normalization._airbyte_raw_nested_s__lting_into_long_names +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..cc143bf47b05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,21 @@ + + + create table + test_normalization.`unnest_alias__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from _airbyte_test_normalization.`unnest_alias_ab3` +-- unnest_alias from test_normalization._airbyte_raw_unnest_alias +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..717bc072a956 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_array__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from _airbyte_test_normalization.`conflict_stream_array_ab3` +-- conflict_stream_array from test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..bdfe747bf5e7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_name__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from _airbyte_test_normalization.`conflict_stream_name_ab3` +-- conflict_stream_name from test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql new file mode 100644 index 000000000000..ded6eda50709 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_name____conflict_stream_name__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + `groups`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from _airbyte_test_normalization.`conflict_stream_name__3flict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name_conflict_stream_name` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..a64da6fc2bba --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_name_conflict_stream_name__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from _airbyte_test_normalization.`conflict_stream_name__2flict_stream_name_ab3` +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from test_normalization.`conflict_stream_name` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..b79e0e4c08a4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`conflict_stream_scalar__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from _airbyte_test_normalization.`conflict_stream_scalar_ab3` +-- conflict_stream_scalar from test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..2c30ff5645d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,22 @@ + + + create table + test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + `DATA`, + `column__'with"_quotes`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_partition_hashid +from _airbyte_test_normalization.`nested_stream_with_co_2g_names_partition_ab3` +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co__lting_into_long_names` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..cc91b90e22c9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_data_hashid +from _airbyte_test_normalization.`nested_stream_with_co_3es_partition_data_ab3` +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql new file mode 100644 index 000000000000..600adbbcf8b8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`nested_stream_with_co__column___with__quotes__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from _airbyte_test_normalization.`nested_stream_with_co_3mn___with__quotes_ab3` +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..a584e5957c67 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from _airbyte_test_normalization.`nested_stream_with_co_3double_array_data_ab3` +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql new file mode 100644 index 000000000000..9b1d5c0c792a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`non_nested_stream_wit__lting_into_long_names__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_non_nested_s__nto_long_names_hashid +from _airbyte_test_normalization.`non_nested_stream_wit_1g_into_long_names_ab3` +-- non_nested_stream_wit__lting_into_long_names from test_normalization._airbyte_raw_non_nest__lting_into_long_names +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..62be462c50f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,21 @@ + + + create table + test_normalization.`unnest_alias_children__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + `owner`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_children_hashid +from _airbyte_test_normalization.`unnest_alias_children_ab3` +-- children at unnest_alias/children from test_normalization.`unnest_alias` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..5747da2ce362 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization.`unnest_alias_children_owner__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_owner_hashid +from _airbyte_test_normalization.`unnest_alias_children_owner_ab3` +-- owner at unnest_alias/children/owner from test_normalization.`unnest_alias_children` +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql new file mode 100644 index 000000000000..8a5111492bbf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -0,0 +1,20 @@ + + + create table + test_normalization_namespace.`simple_stream_with_na__lting_into_long_names__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + `date`, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_simple_strea__nto_long_names_hashid +from _airbyte_test_normalization_namespace.`simple_stream_with_na_1g_into_long_names_ab3` +-- simple_stream_with_na__lting_into_long_names from test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql new file mode 100644 index 000000000000..f8c6c82e2fbc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_co_1g_into_long_names_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(`partition` as char), '')) as char)) as _airbyte_nested_strea__nto_long_names_hashid, + tmp.* +from _airbyte_test_normalization.`nested_stream_with_co_1g_into_long_names_ab2` tmp +-- nested_stream_with_co__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index e65c6a32e65a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_cdc_excluded_ab4__dbt_tmp` as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from _airbyte_test_normalization.`dedup_cdc_excluded_ab3` tmp --- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index 6e7a324f9a3d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_ab4__dbt_tmp` as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from _airbyte_test_normalization.`dedup_exchange_rate_ab3` tmp --- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index f526a98321b9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view _airbyte_test_normalization.`pos_dedup_cdcx_ab4__dbt_tmp` as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from _airbyte_test_normalization.`pos_dedup_cdcx_ab3` tmp --- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 900871e7e77a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,29 +0,0 @@ - - - create table - test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - `name`, - `column__'with"_quotes`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from _airbyte_test_normalization.`dedup_cdc_excluded_ab4` --- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 7376aaa375be..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,31 +0,0 @@ - - - create table - test_normalization.`dedup_exchange_rate_scd__dbt_tmp` - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - `date`, - timestamp_col, - `HKD@spéçiäl & characters`, - hkd_special___characters, - nzd, - usd, - `date` as _airbyte_start_at, - lag(`date`) over ( - partition by id, currency, cast(nzd as char) - order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(`date`) over ( - partition by id, currency, cast(nzd as char) - order by `date` is null asc, `date` desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from _airbyte_test_normalization.`dedup_exchange_rate_ab4` --- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 9712b7bb96aa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,29 +0,0 @@ - - - create table - test_normalization.`pos_dedup_cdcx_scd__dbt_tmp` - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - `name`, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from _airbyte_test_normalization.`pos_dedup_cdcx_ab4` --- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_row_num = 1 - ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 6fd2103d5692..33bc1c9a474e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -8,14 +8,18 @@ select json_value(_airbyte_data, '$."name"') as `name`, json_value(_airbyte_data, - '$."column___with__quotes"') as `column__'with"_quotes`, - json_value(_airbyte_data, '$."_ab_cdc_lsn"') as _ab_cdc_lsn, json_value(_airbyte_data, '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, json_value(_airbyte_data, '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias -- dedup_cdc_excluded +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index 394e3a59e8d7..c725452b87b1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -7,7 +7,6 @@ select signed ) as id, cast(`name` as char) as `name`, - cast(`column__'with"_quotes` as char) as `column__'with"_quotes`, cast(_ab_cdc_lsn as float ) as _ab_cdc_lsn, @@ -17,7 +16,13 @@ select cast(_ab_cdc_deleted_at as float ) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`dedup_cdc_excluded_ab1` -- dedup_cdc_excluded +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index dae5bd72e4ab..e42ac40db163 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -19,7 +19,13 @@ select '$."NZD"') as nzd, json_value(_airbyte_data, '$."USD"') as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index a935dd0e7021..489196399610 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -22,7 +22,13 @@ select cast(usd as float ) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`dedup_exchange_rate_ab1` -- dedup_exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index b8662205d239..335aa00cf89f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -19,7 +19,13 @@ select '$."NZD"') as nzd, json_value(_airbyte_data, '$."USD"') as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization._airbyte_raw_exchange_rate as table_alias -- exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql similarity index 86% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 081f20d75019..258e3f4caf08 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -22,7 +22,13 @@ select cast(usd as float ) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`exchange_rate_ab1` -- exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 85d7fb2288bc..6cc3a26b097f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -7,4 +7,6 @@ select tmp.* from _airbyte_test_normalization.`exchange_rate_ab2` tmp -- exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql similarity index 86% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 16b2b48b4ca4..7fd41343ac43 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -15,7 +15,12 @@ select '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, json_value(_airbyte_data, '$."_ab_cdc_log_pos"') as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias -- pos_dedup_cdcx +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index 50eec775907d..9f8c55f05dcf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -19,7 +19,12 @@ select cast(_ab_cdc_log_pos as float ) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at from _airbyte_test_normalization.`pos_dedup_cdcx_ab1` -- pos_dedup_cdcx +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..446b1c167a26 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,17 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..01fe01259f91 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..4dd4da0f3431 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,74 @@ + + + create table + test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`dedup_cdc_excluded_ab3` + -- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..03eb062ccd38 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,80 @@ + + + create table + test_normalization.`dedup_exchange_rate_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`dedup_exchange_rate_ab3` + -- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(nzd as char), '')) as char)) as _airbyte_unique_key, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + `date` as _airbyte_start_at, + lag(`date`) over ( + partition by id, currency, cast(nzd as char) + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(`date`) over ( + partition by id, currency, cast(nzd as char) + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..13bbe0a227a9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,66 @@ + + + create table + test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + -- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index eb404ddd52a8..37503d5bb636 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -6,15 +6,21 @@ -- Final base SQL model select + _airbyte_unique_key, id, `name`, - `column__'with"_quotes`, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_dedup_cdc_excluded_hashid from test_normalization.`dedup_cdc_excluded_scd` -- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index d38a5c453ae5..c5ad30dc78c0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -6,6 +6,7 @@ -- Final base SQL model select + _airbyte_unique_key, id, currency, `date`, @@ -14,9 +15,15 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from test_normalization.`dedup_exchange_rate_scd` -- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql index e24cf0e7d279..6b8a3751184f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -14,8 +14,14 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from _airbyte_test_normalization.`exchange_rate_ab3` -- exchange_rate from test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..10a547a0cccb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,22 @@ + + + create table + test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from test_normalization.`renamed_dedup_cdc_excluded_scd` +-- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..be6a9a952a26 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,76 @@ + + + create table + test_normalization.`pos_dedup_cdcx_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`pos_dedup_cdcx_ab3` + -- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char), cast(_ab_cdc_log_pos as char) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index e166fc1d244e..736d35c45a9a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -6,15 +6,21 @@ -- Final base SQL model select + _airbyte_unique_key, id, `name`, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from test_normalization.`pos_dedup_cdcx_scd` -- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql similarity index 53% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql index ec1da44389ca..3caccb347235 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -3,8 +3,10 @@ -- SQL model to build a hash column based on the values of this record select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), '')) as char)) as _airbyte_dedup_cdc_excluded_hashid, + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), '')) as char)) as _airbyte_dedup_cdc_excluded_hashid, tmp.* from _airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp -- dedup_cdc_excluded +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 83a9b9726049..98129db90185 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -7,4 +7,6 @@ select tmp.* from _airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp -- dedup_exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index 6d71da37893e..857c87e6fc1d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -7,4 +7,5 @@ select tmp.* from _airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp -- pos_dedup_cdcx +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..2df6fcdc3715 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 6144a13617e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 6d694c280ee7..5fb29052e85b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as {{ adapter.quote('name') }}, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column__\'with"_quotes') }}, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index dc225106de96..42bbbbc98de8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, cast({{ adapter.quote('name') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('name') }}, - cast({{ adapter.quote('column__\'with"_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column__\'with"_quotes') }}, cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index aa34179c3c5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index c9a6865ef7fd..5ed2b660ee09 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 4690ff9a33f1..a45b5767c899 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -12,7 +16,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index ef5f6e52d60e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 3e66f6d0b4fc..c20f382cfbc0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index de09dc96002c..733803bde010 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -12,7 +16,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 962cb57a8666..c645234a8db9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +18,6 @@ select tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 1bd874551007..ac502fab3a57 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -7,7 +11,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index c132580004d5..f6664f48d0bb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -7,7 +11,10 @@ select cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 7c897124b395..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..7fd3046082a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..a04f92bdcd55 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..ff4320b54f64 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,111 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..470401fb5740 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,121 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..b4ae7c90fe73 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,103 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql similarity index 52% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index 2152d113d03a..af2a08ea048f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -1,15 +1,23 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, {{ adapter.quote('name') }}, - {{ adapter.quote('column__\'with"_quotes') }}, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_cdc_excluded_hashid from {{ ref('dedup_cdc_excluded_scd') }} -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 57% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index f10a9446d7b8..06a37a760378 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,6 +1,11 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, currency, {{ adapter.quote('date') }}, @@ -9,9 +14,13 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from {{ ref('dedup_exchange_rate_scd') }} -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql similarity index 57% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql index 147399e6739d..980126a4146c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, @@ -9,8 +13,12 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from {{ ref('exchange_rate_ab3') }} -- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..8a8ff85f5902 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,19 @@ +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..7a3a501bb98f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,113 @@ +{{ config( + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 8d417147dfdd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - {{ adapter.quote('name') }}, - {{ adapter.quote('column__\'with"_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 739992eb7d2e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index ae62dc05f870..1a756d5b7507 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,23 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, {{ adapter.quote('name') }}, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index c25354e4fba4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql index 67de7e9893fb..6d34fc802057 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -1,10 +1,13 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ 'id', adapter.quote('name'), - adapter.quote('column__\'with"_quotes'), '_ab_cdc_lsn', '_ab_cdc_updated_at', '_ab_cdc_deleted_at', @@ -12,4 +15,6 @@ select tmp.* from {{ ref('dedup_cdc_excluded_ab2') }} tmp -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 65% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 3e68860804dc..9221c1833031 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +18,6 @@ select tmp.* from {{ ref('dedup_exchange_rate_ab2') }} tmp -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 67% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index 5b9a513b2e86..78def1d5d1aa 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -12,4 +16,5 @@ select tmp.* from {{ ref('pos_dedup_cdcx_ab2') }} tmp -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..f5ce3e6ba182 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml index c085ca8ccc25..45c338b893ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _airbyte_raw_dedup_exchange_rate - name: _airbyte_raw_exchange_rate - name: _airbyte_raw_pos_dedup_cdcx + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..33bc1c9a474e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,25 @@ + + create view _airbyte_test_normalization.`dedup_cdc_excluded_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."name"') as `name`, + json_value(_airbyte_data, + '$."_ab_cdc_lsn"') as _ab_cdc_lsn, + json_value(_airbyte_data, + '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, + json_value(_airbyte_data, + '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..c725452b87b1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ + + create view _airbyte_test_normalization.`dedup_cdc_excluded_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(`name` as char) as `name`, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`dedup_cdc_excluded_ab1` +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..e42ac40db163 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,31 @@ + + create view _airbyte_test_normalization.`dedup_exchange_rate_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."currency"') as currency, + json_value(_airbyte_data, + '$."date"') as `date`, + json_value(_airbyte_data, + '$."timestamp_col"') as timestamp_col, + json_value(_airbyte_data, + '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + json_value(_airbyte_data, + '$."HKD_special___characters"') as hkd_special___characters, + json_value(_airbyte_data, + '$."NZD"') as nzd, + json_value(_airbyte_data, + '$."USD"') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..489196399610 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,34 @@ + + create view _airbyte_test_normalization.`dedup_exchange_rate_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(currency as char) as currency, + case when `date` = '' then NULL + else cast(`date` as date) + end as `date` + , + cast(nullif(timestamp_col, '') as char) as timestamp_col, + cast(`HKD@spéçiäl & characters` as + float +) as `HKD@spéçiäl & characters`, + cast(hkd_special___characters as char) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`dedup_exchange_rate_ab1` +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..335aa00cf89f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,31 @@ + + create view _airbyte_test_normalization.`exchange_rate_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."currency"') as currency, + json_value(_airbyte_data, + '$."date"') as `date`, + json_value(_airbyte_data, + '$."timestamp_col"') as timestamp_col, + json_value(_airbyte_data, + '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + json_value(_airbyte_data, + '$."HKD_special___characters"') as hkd_special___characters, + json_value(_airbyte_data, + '$."NZD"') as nzd, + json_value(_airbyte_data, + '$."USD"') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..258e3f4caf08 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,34 @@ + + create view _airbyte_test_normalization.`exchange_rate_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(currency as char) as currency, + case when `date` = '' then NULL + else cast(`date` as date) + end as `date` + , + cast(nullif(timestamp_col, '') as char) as timestamp_col, + cast(`HKD@spéçiäl & characters` as + float +) as `HKD@spéçiäl & characters`, + cast(hkd_special___characters as char) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`exchange_rate_ab1` +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..6cc3a26b097f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`exchange_rate_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_exchange_rate_hashid, + tmp.* +from _airbyte_test_normalization.`exchange_rate_ab2` tmp +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..7fd41343ac43 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,26 @@ + + create view _airbyte_test_normalization.`pos_dedup_cdcx_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + json_value(_airbyte_data, + '$."name"') as `name`, + json_value(_airbyte_data, + '$."_ab_cdc_lsn"') as _ab_cdc_lsn, + json_value(_airbyte_data, + '$."_ab_cdc_updated_at"') as _ab_cdc_updated_at, + json_value(_airbyte_data, + '$."_ab_cdc_deleted_at"') as _ab_cdc_deleted_at, + json_value(_airbyte_data, + '$."_ab_cdc_log_pos"') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..9f8c55f05dcf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,30 @@ + + create view _airbyte_test_normalization.`pos_dedup_cdcx_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + cast(`name` as char) as `name`, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`pos_dedup_cdcx_ab1` +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..446b1c167a26 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,17 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1__dbt_tmp` as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, + '$."id"') as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..01fe01259f91 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,18 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2__dbt_tmp` as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + signed +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at +from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab1` +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..4dd4da0f3431 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,74 @@ + + + create table + test_normalization.`dedup_cdc_excluded_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`dedup_cdc_excluded_ab3` + -- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..03eb062ccd38 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,80 @@ + + + create table + test_normalization.`dedup_exchange_rate_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`dedup_exchange_rate_ab3` + -- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(nzd as char), '')) as char)) as _airbyte_unique_key, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + `date` as _airbyte_start_at, + lag(`date`) over ( + partition by id, currency, cast(nzd as char) + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(`date`) over ( + partition by id, currency, cast(nzd as char) + order by + `date` is null asc, + `date` desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..13bbe0a227a9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,66 @@ + + + create table + test_normalization.`renamed_dedup_cdc_excluded_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3` + -- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..37503d5bb636 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,26 @@ + + + create table + test_normalization.`dedup_cdc_excluded__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from test_normalization.`dedup_cdc_excluded_scd` +-- dedup_cdc_excluded from test_normalization._airbyte_raw_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..c5ad30dc78c0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,29 @@ + + + create table + test_normalization.`dedup_exchange_rate__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from test_normalization.`dedup_exchange_rate_scd` +-- dedup_exchange_rate from test_normalization._airbyte_raw_dedup_exchange_rate +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..6b8a3751184f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,27 @@ + + + create table + test_normalization.`exchange_rate__dbt_tmp` + as ( + +-- Final base SQL model +select + id, + currency, + `date`, + timestamp_col, + `HKD@spéçiäl & characters`, + hkd_special___characters, + nzd, + usd, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from _airbyte_test_normalization.`exchange_rate_ab3` +-- exchange_rate from test_normalization._airbyte_raw_exchange_rate +where 1 = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..10a547a0cccb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,22 @@ + + + create table + test_normalization.`renamed_dedup_cdc_excluded__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from test_normalization.`renamed_dedup_cdc_excluded_scd` +-- renamed_dedup_cdc_excluded from test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..be6a9a952a26 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,76 @@ + + + create table + test_normalization.`pos_dedup_cdcx_scd__dbt_tmp` + as ( + +with + +input_data as ( + select * + from _airbyte_test_normalization.`pos_dedup_cdcx_ab3` + -- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as char), cast(_ab_cdc_updated_at as char), cast(_ab_cdc_log_pos as char) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(concat(coalesce(cast(_airbyte_unique_key as char), ''), '-', coalesce(cast(_airbyte_start_at as char), ''), '-', coalesce(cast(_airbyte_emitted_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..736d35c45a9a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,26 @@ + + + create table + test_normalization.`pos_dedup_cdcx__dbt_tmp` + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + `name`, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + + CURRENT_TIMESTAMP + as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from test_normalization.`pos_dedup_cdcx_scd` +-- pos_dedup_cdcx from test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..3caccb347235 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`dedup_cdc_excluded_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), '')) as char)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from _airbyte_test_normalization.`dedup_cdc_excluded_ab2` tmp +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..98129db90185 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`dedup_exchange_rate_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from _airbyte_test_normalization.`dedup_exchange_rate_ab2` tmp +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..857c87e6fc1d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,11 @@ + + create view _airbyte_test_normalization.`pos_dedup_cdcx_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`name` as char), ''), '-', coalesce(cast(_ab_cdc_lsn as char), ''), '-', coalesce(cast(_ab_cdc_updated_at as char), ''), '-', coalesce(cast(_ab_cdc_deleted_at as char), ''), '-', coalesce(cast(_ab_cdc_log_pos as char), '')) as char)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from _airbyte_test_normalization.`pos_dedup_cdcx_ab2` tmp +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..2df6fcdc3715 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,12 @@ + + create view _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab3__dbt_tmp` as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(concat(coalesce(cast(id as char), '')) as char)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from _airbyte_test_normalization.`renamed_dedup_cdc_excluded_ab2` tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 14f757f19bcf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,100 +0,0 @@ - - - create table test_normalization.dedup_cdc_excluded_scd__dbt_tmp - - as - -with dbt__cte__dedup_cdc_excluded_ab1__ as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - json_value("_AIRBYTE_DATA", '$."id"') as id, - json_value("_AIRBYTE_DATA", '$."name"') as name, - json_value("_AIRBYTE_DATA", '$."column___with__quotes"') as column___with__quotes, - json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", - json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", - "_AIRBYTE_EMITTED_AT" -from test_normalization.airbyte_raw_dedup_cdc_excluded --- dedup_cdc_excluded -), dbt__cte__dedup_cdc_excluded_ab2__ as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - numeric -) as id, - cast(name as varchar2(4000)) as name, - cast(column___with__quotes as varchar2(4000)) as column___with__quotes, - cast("_AB_CDC_LSN" as - float -) as "_AB_CDC_LSN", - cast("_AB_CDC_UPDATED_AT" as - float -) as "_AB_CDC_UPDATED_AT", - cast("_AB_CDC_DELETED_AT" as - float -) as "_AB_CDC_DELETED_AT", - "_AIRBYTE_EMITTED_AT" -from dbt__cte__dedup_cdc_excluded_ab1__ --- dedup_cdc_excluded -), dbt__cte__dedup_cdc_excluded_ab3__ as ( - --- SQL model to build a hash column based on the values of this record -select - ora_hash( - - id || '~' || - - - name || '~' || - - - column___with__quotes || '~' || - - - "_AB_CDC_LSN" || '~' || - - - "_AB_CDC_UPDATED_AT" || '~' || - - - "_AB_CDC_DELETED_AT" - - ) as "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID", - tmp.* -from dbt__cte__dedup_cdc_excluded_ab2__ tmp --- dedup_cdc_excluded -), dbt__cte__dedup_cdc_excluded_ab4__ as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" - order by "_AIRBYTE_EMITTED_AT" asc - ) as "_AIRBYTE_ROW_NUM", - tmp.* -from dbt__cte__dedup_cdc_excluded_ab3__ tmp --- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - column___with__quotes, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", - lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by "_AIRBYTE_EMITTED_AT" asc nulls last, "_AIRBYTE_EMITTED_AT" desc, "_AIRBYTE_EMITTED_AT" desc - ) as "_AIRBYTE_END_AT", - case when lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by "_AIRBYTE_EMITTED_AT" asc nulls last, "_AIRBYTE_EMITTED_AT" desc, "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc - ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" -from dbt__cte__dedup_cdc_excluded_ab4__ --- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded -where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..bd51b214ebc6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,93 @@ + + + create table test_normalization.dedup_cdc_excluded_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.dedup_cdc_excluded_ab3 + -- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc + ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)) + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_UPDATED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..e6c91980fea5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,99 @@ + + + create table test_normalization.dedup_exchange_rate_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.dedup_exchange_rate_ab3 + -- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id || '~' || + + + currency || '~' || + + + nzd + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "DATE" as "_AIRBYTE_START_AT", + lag("DATE") over ( + partition by id, currency, cast(nzd as varchar2(4000)) + order by + "DATE" asc nulls last, + "DATE" desc, + "_AIRBYTE_EMITTED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("DATE") over ( + partition by id, currency, cast(nzd as varchar2(4000)) + order by + "DATE" asc nulls last, + "DATE" desc, + "_AIRBYTE_EMITTED_AT" desc + ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..c8d9914d17d2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,79 @@ + + + create table test_normalization.renamed_dedup_cdc_excluded_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.renamed_dedup_cdc_excluded_ab3 + -- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc + ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql similarity index 72% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index d55edca65a1f..bf4c8a11c07b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -6,14 +6,19 @@ -- Final base SQL model select + "_AIRBYTE_UNIQUE_KEY", id, name, - column___with__quotes, "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" from test_normalization.dedup_cdc_excluded_scd -- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded -where "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 74% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 72b753fb7e87..f67f7384cf17 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -6,6 +6,7 @@ -- Final base SQL model select + "_AIRBYTE_UNIQUE_KEY", id, currency, "DATE", @@ -14,8 +15,13 @@ select hkd_special___characters_1, nzd, usd, + "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" from test_normalization.dedup_exchange_rate_scd -- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate -where "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql index ee3cc52e8b43..9d77656b65a9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -16,9 +16,15 @@ select json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, json_value("_AIRBYTE_DATA", '$."USD"') as usd, - "_AIRBYTE_EMITTED_AT" + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" from test_normalization.airbyte_raw_exchange_rate -- exchange_rate +where 1 = 1 + ), dbt__cte__exchange_rate_ab2__ as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -43,9 +49,15 @@ select cast(usd as float ) as usd, - "_AIRBYTE_EMITTED_AT" + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" from dbt__cte__exchange_rate_ab1__ -- exchange_rate +where 1 = 1 + ), dbt__cte__exchange_rate_ab3__ as ( -- SQL model to build a hash column based on the values of this record @@ -79,6 +91,8 @@ select tmp.* from dbt__cte__exchange_rate_ab2__ tmp -- exchange_rate +where 1 = 1 + )-- Final base SQL model select id, @@ -89,7 +103,12 @@ select hkd_special___characters_1, nzd, usd, + "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_EXCHANGE_RATE_HASHID" from dbt__cte__exchange_rate_ab3__ --- exchange_rate from test_normalization.airbyte_raw_exchange_rate \ No newline at end of file +-- exchange_rate from test_normalization.airbyte_raw_exchange_rate +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..feac3368a6ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,20 @@ + + + create table test_normalization.renamed_dedup_cdc_excluded__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" +from test_normalization.renamed_dedup_cdc_excluded_scd +-- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..c184fd31e2d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,98 @@ + + + create table test_normalization.pos_dedup_cdcx_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.pos_dedup_cdcx_ab3 + -- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc + ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)), cast("_AB_CDC_LOG_POS" as varchar2(4000)) + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_LOG_POS" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql similarity index 72% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index 48197614eaaa..d3fd87ee5576 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -6,14 +6,20 @@ -- Final base SQL model select + "_AIRBYTE_UNIQUE_KEY", id, name, "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AB_CDC_LOG_POS", + "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_POS_DEDUP_CDCX_HASHID" from test_normalization.pos_dedup_cdcx_scd -- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx -where "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..30135e3c86ec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,72 @@ + + create view test_normalization.dedup_cdc_excluded_ab3__dbt_tmp as + +with dbt__cte__dedup_cdc_excluded_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."name"') as name, + json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_dedup_cdc_excluded +-- dedup_cdc_excluded +where 1 = 1 + +), dbt__cte__dedup_cdc_excluded_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(name as varchar2(4000)) as name, + cast("_AB_CDC_LSN" as + float +) as "_AB_CDC_LSN", + cast("_AB_CDC_UPDATED_AT" as + float +) as "_AB_CDC_UPDATED_AT", + cast("_AB_CDC_DELETED_AT" as + float +) as "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__dedup_cdc_excluded_ab1__ +-- dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + name || '~' || + + + "_AB_CDC_LSN" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" + + ) as "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID", + tmp.* +from dbt__cte__dedup_cdc_excluded_ab2__ tmp +-- dedup_cdc_excluded +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 57% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 16dd125da9bb..527196251304 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,8 +1,5 @@ - - create table test_normalization.dedup_exchange_rate_scd__dbt_tmp - - as + create view test_normalization.dedup_exchange_rate_ab3__dbt_tmp as with dbt__cte__dedup_exchange_rate_ab1__ as ( @@ -16,9 +13,15 @@ select json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, json_value("_AIRBYTE_DATA", '$."USD"') as usd, - "_AIRBYTE_EMITTED_AT" + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" from test_normalization.airbyte_raw_dedup_exchange_rate -- dedup_exchange_rate +where 1 = 1 + ), dbt__cte__dedup_exchange_rate_ab2__ as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -43,12 +46,16 @@ select cast(usd as float ) as usd, - "_AIRBYTE_EMITTED_AT" + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" from dbt__cte__dedup_exchange_rate_ab1__ -- dedup_exchange_rate -), dbt__cte__dedup_exchange_rate_ab3__ as ( +where 1 = 1 --- SQL model to build a hash column based on the values of this record +)-- SQL model to build a hash column based on the values of this record select ora_hash( @@ -79,38 +86,6 @@ select tmp.* from dbt__cte__dedup_exchange_rate_ab2__ tmp -- dedup_exchange_rate -), dbt__cte__dedup_exchange_rate_ab4__ as ( +where 1 = 1 + --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" - order by "_AIRBYTE_EMITTED_AT" asc - ) as "_AIRBYTE_ROW_NUM", - tmp.* -from dbt__cte__dedup_exchange_rate_ab3__ tmp --- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - "DATE", - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - "DATE" as "_AIRBYTE_START_AT", - lag("DATE") over ( - partition by id, currency, cast(nzd as varchar2(4000)) - order by "DATE" asc nulls last, "DATE" desc, "_AIRBYTE_EMITTED_AT" desc - ) as "_AIRBYTE_END_AT", - case when lag("DATE") over ( - partition by id, currency, cast(nzd as varchar2(4000)) - order by "DATE" asc nulls last, "DATE" desc, "_AIRBYTE_EMITTED_AT" desc - ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" -from dbt__cte__dedup_exchange_rate_ab4__ --- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate -where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 51% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index be9d106c3db0..e1a0af83a632 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -1,8 +1,5 @@ - - create table test_normalization.pos_dedup_cdcx_scd__dbt_tmp - - as + create view test_normalization.pos_dedup_cdcx_ab3__dbt_tmp as with dbt__cte__pos_dedup_cdcx_ab1__ as ( @@ -14,9 +11,14 @@ select json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", json_value("_AIRBYTE_DATA", '$."_ab_cdc_log_pos"') as "_AB_CDC_LOG_POS", - "_AIRBYTE_EMITTED_AT" + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" from test_normalization.airbyte_raw_pos_dedup_cdcx -- pos_dedup_cdcx +where 1 = 1 ), dbt__cte__pos_dedup_cdcx_ab2__ as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -37,12 +39,15 @@ select cast("_AB_CDC_LOG_POS" as float ) as "_AB_CDC_LOG_POS", - "_AIRBYTE_EMITTED_AT" + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" from dbt__cte__pos_dedup_cdcx_ab1__ -- pos_dedup_cdcx -), dbt__cte__pos_dedup_cdcx_ab3__ as ( - --- SQL model to build a hash column based on the values of this record +where 1 = 1 +)-- SQL model to build a hash column based on the values of this record select ora_hash( @@ -67,36 +72,5 @@ select tmp.* from dbt__cte__pos_dedup_cdcx_ab2__ tmp -- pos_dedup_cdcx -), dbt__cte__pos_dedup_cdcx_ab4__ as ( +where 1 = 1 --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by "_AIRBYTE_POS_DEDUP_CDCX_HASHID" - order by "_AIRBYTE_EMITTED_AT" asc - ) as "_AIRBYTE_ROW_NUM", - tmp.* -from dbt__cte__pos_dedup_cdcx_ab3__ tmp --- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - "_AB_CDC_LSN", - "_AB_CDC_UPDATED_AT", - "_AB_CDC_DELETED_AT", - "_AB_CDC_LOG_POS", - "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", - lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by "_AIRBYTE_EMITTED_AT" asc nulls last, "_AIRBYTE_EMITTED_AT" desc, "_AIRBYTE_EMITTED_AT" desc - ) as "_AIRBYTE_END_AT", - case when lag("_AIRBYTE_EMITTED_AT") over ( - partition by id - order by "_AIRBYTE_EMITTED_AT" asc nulls last, "_AIRBYTE_EMITTED_AT" desc, "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc - ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", - "_AIRBYTE_EMITTED_AT", - "_AIRBYTE_POS_DEDUP_CDCX_HASHID" -from dbt__cte__pos_dedup_cdcx_ab4__ --- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx -where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..6ad67cd02a90 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,46 @@ + + create view test_normalization.renamed_dedup_cdc_excluded_ab3__dbt_tmp as + +with dbt__cte__renamed_dedup_cdc_excluded_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), dbt__cte__renamed_dedup_cdc_excluded_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__renamed_dedup_cdc_excluded_ab1__ +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id + + ) as "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID", + tmp.* +from dbt__cte__renamed_dedup_cdc_excluded_ab2__ tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index fa2cc2ec8b0b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', 'airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index b96dabd43418..2f3b6d1b7104 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,20 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['name'], ['name']) }} as name, - {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as column___with__quotes, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as {{ quote('_AB_CDC_LSN') }}, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as {{ quote('_AB_CDC_UPDATED_AT') }}, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index 00175dd65806..9f7cb197462d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,20 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, cast(name as {{ dbt_utils.type_string() }}) as name, - cast(column___with__quotes as {{ dbt_utils.type_string() }}) as column___with__quotes, cast({{ quote('_AB_CDC_LSN') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_LSN') }}, cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_UPDATED_AT') }}, cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index 7991b32d4157..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc - ) as {{ quote('_AIRBYTE_ROW_NUM') }}, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index e71a865c767e..a0dda141cd02 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters_1, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['USD'], ['USD']) }} as usd, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index e79c6d54210d..fadc49c40be3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +13,11 @@ select cast(hkd_special___characters_1 as {{ dbt_utils.type_string() }}) as hkd_special___characters_1, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index 28d9f65311da..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc - ) as {{ quote('_AIRBYTE_ROW_NUM') }}, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 130eb4e0e636..bcc1db00d8e3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, @@ -9,7 +13,11 @@ select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters_1, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['USD'], ['USD']) }} as usd, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ source('test_normalization', 'airbyte_raw_exchange_rate') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 63adb62a688e..e7f5e3353cfd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +13,11 @@ select cast(hkd_special___characters_1 as {{ dbt_utils.type_string() }}) as hkd_special___characters_1, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ ref('exchange_rate_ab1') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 2cf009488395..21fb107c1c44 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +18,6 @@ select tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 3cc4f3f73faf..9cb45ceefec4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, @@ -7,7 +11,10 @@ select {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as {{ quote('_AB_CDC_UPDATED_AT') }}, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as {{ quote('_AB_CDC_DELETED_AT') }}, {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index 344413a6b470..de537cd43138 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -7,7 +11,10 @@ select cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_UPDATED_AT') }}, cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_DELETED_AT') }}, cast({{ quote('_AB_CDC_LOG_POS') }} as {{ dbt_utils.type_float() }}) as {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 8734f4c27df5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc - ) as {{ quote('_AIRBYTE_ROW_NUM') }}, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..e7fe7d6f462c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar(quote('_AIRBYTE_DATA'), ['id'], ['id']) }} as id, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} +from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..fc3191e47f30 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }} +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..3f81f2997f7c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,111 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} +), +new_data_ids as ( + -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + name, + {{ quote('_AB_CDC_LSN') }}, + {{ quote('_AB_CDC_UPDATED_AT') }}, + {{ quote('_AB_CDC_DELETED_AT') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, + lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc + ) as {{ quote('_AIRBYTE_END_AT') }}, + case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc + ) is null and {{ quote('_AB_CDC_DELETED_AT') }} is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_string() }}), cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_string() }}) + order by {{ quote('_AIRBYTE_AB_ID') }} + ) as {{ quote('_AIRBYTE_ROW_NUM') }}, + {{ dbt_utils.surrogate_key([ + quote('_AIRBYTE_UNIQUE_KEY'), + quote('_AIRBYTE_START_AT'), + quote('_AIRBYTE_EMITTED_AT'), quote('_AB_CDC_DELETED_AT'), quote('_AB_CDC_UPDATED_AT') + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + scd_data.* + from scd_data +) +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + id, + name, + {{ quote('_AB_CDC_LSN') }}, + {{ quote('_AB_CDC_UPDATED_AT') }}, + {{ quote('_AB_CDC_DELETED_AT') }}, + {{ quote('_AIRBYTE_START_AT') }}, + {{ quote('_AIRBYTE_END_AT') }}, + {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} +from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..6ad3983885b4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,121 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} +), +new_data_ids as ( + -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + currency, + {{ quote('DATE') }}, + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + {{ quote('DATE') }} as {{ quote('_AIRBYTE_START_AT') }}, + lag({{ quote('DATE') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ quote('DATE') }} asc nulls last, + {{ quote('DATE') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc + ) as {{ quote('_AIRBYTE_END_AT') }}, + case when lag({{ quote('DATE') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ quote('DATE') }} asc nulls last, + {{ quote('DATE') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc + ) is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }} + order by {{ quote('_AIRBYTE_AB_ID') }} + ) as {{ quote('_AIRBYTE_ROW_NUM') }}, + {{ dbt_utils.surrogate_key([ + quote('_AIRBYTE_UNIQUE_KEY'), + quote('_AIRBYTE_START_AT'), + quote('_AIRBYTE_EMITTED_AT') + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + scd_data.* + from scd_data +) +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + id, + currency, + {{ quote('DATE') }}, + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + {{ quote('_AIRBYTE_START_AT') }}, + {{ quote('_AIRBYTE_END_AT') }}, + {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} +from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..9d79d2cbac72 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,103 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} +), +new_data_ids as ( + -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, + lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc + ) as {{ quote('_AIRBYTE_END_AT') }}, + case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc + ) is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }} + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }} + order by {{ quote('_AIRBYTE_AB_ID') }} + ) as {{ quote('_AIRBYTE_ROW_NUM') }}, + {{ dbt_utils.surrogate_key([ + quote('_AIRBYTE_UNIQUE_KEY'), + quote('_AIRBYTE_START_AT'), + quote('_AIRBYTE_EMITTED_AT') + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + scd_data.* + from scd_data +) +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + id, + {{ quote('_AIRBYTE_START_AT') }}, + {{ quote('_AIRBYTE_END_AT') }}, + {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }} +from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql similarity index 50% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index 9578d423ecf4..20be5269fbc9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -1,15 +1,23 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, id, name, - column___with__quotes, {{ quote('_AB_CDC_LSN') }}, {{ quote('_AB_CDC_UPDATED_AT') }}, {{ quote('_AB_CDC_DELETED_AT') }}, + {{ quote('_AIRBYTE_AB_ID') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} from {{ ref('dedup_cdc_excluded_scd') }} -- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} -where "_AIRBYTE_ACTIVE_ROW" = 1 +where 1 = 1 +and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 51% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index d423d5bf7f1a..d130087cf48a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,6 +1,11 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, id, currency, {{ quote('DATE') }}, @@ -9,9 +14,13 @@ select hkd_special___characters_1, nzd, usd, + {{ quote('_AIRBYTE_AB_ID') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} from {{ ref('dedup_exchange_rate_scd') }} -- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} -where "_AIRBYTE_ACTIVE_ROW" = 1 +where 1 = 1 +and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql similarity index 53% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql index 5ca8bf28bd17..c87e205f2a32 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, @@ -9,8 +13,12 @@ select hkd_special___characters_1, nzd, usd, + {{ quote('_AIRBYTE_AB_ID') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, {{ quote('_AIRBYTE_EXCHANGE_RATE_HASHID') }} from {{ ref('exchange_rate_ab3') }} -- exchange_rate from {{ source('test_normalization', 'airbyte_raw_exchange_rate') }} +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..40bf1d0d0013 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,19 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }} +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..506d15538add --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,113 @@ +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} +), +new_data_ids as ( + -- build a subset of {{ quote('_AIRBYTE_UNIQUE_KEY') }} from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }} + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data.{{ quote('_AIRBYTE_UNIQUE_KEY') }} = new_data_ids.{{ quote('_AIRBYTE_UNIQUE_KEY') }} + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + id, + name, + {{ quote('_AB_CDC_LSN') }}, + {{ quote('_AB_CDC_UPDATED_AT') }}, + {{ quote('_AB_CDC_DELETED_AT') }}, + {{ quote('_AB_CDC_LOG_POS') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, + lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc, {{ quote('_AB_CDC_LOG_POS') }} desc + ) as {{ quote('_AIRBYTE_END_AT') }}, + case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( + partition by id + order by + {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, + {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc, {{ quote('_AB_CDC_LOG_POS') }} desc + ) is null and {{ quote('_AB_CDC_DELETED_AT') }} is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by {{ quote('_AIRBYTE_UNIQUE_KEY') }}, {{ quote('_AIRBYTE_START_AT') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, cast({{ quote('_AB_CDC_DELETED_AT') }} as {{ dbt_utils.type_string() }}), cast({{ quote('_AB_CDC_UPDATED_AT') }} as {{ dbt_utils.type_string() }}), cast({{ quote('_AB_CDC_LOG_POS') }} as {{ dbt_utils.type_string() }}) + order by {{ quote('_AIRBYTE_AB_ID') }} + ) as {{ quote('_AIRBYTE_ROW_NUM') }}, + {{ dbt_utils.surrogate_key([ + quote('_AIRBYTE_UNIQUE_KEY'), + quote('_AIRBYTE_START_AT'), + quote('_AIRBYTE_EMITTED_AT'), quote('_AB_CDC_DELETED_AT'), quote('_AB_CDC_UPDATED_AT'), quote('_AB_CDC_LOG_POS') + ]) }} as {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + scd_data.* + from scd_data +) +select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, + {{ quote('_AIRBYTE_UNIQUE_KEY_SCD') }}, + id, + name, + {{ quote('_AB_CDC_LSN') }}, + {{ quote('_AB_CDC_UPDATED_AT') }}, + {{ quote('_AB_CDC_DELETED_AT') }}, + {{ quote('_AB_CDC_LOG_POS') }}, + {{ quote('_AIRBYTE_START_AT') }}, + {{ quote('_AIRBYTE_END_AT') }}, + {{ quote('_AIRBYTE_ACTIVE_ROW') }}, + {{ quote('_AIRBYTE_AB_ID') }}, + {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, + {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} +from dedup_data where {{ quote('_AIRBYTE_ROW_NUM') }} = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 167dd8604a46..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - column___with__quotes, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, - lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) as {{ quote('_AIRBYTE_END_AT') }}, - case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc - ) is null and {{ quote('_AB_CDC_DELETED_AT') }} is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID') }} -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', 'airbyte_raw_dedup_cdc_excluded') }} -where "_AIRBYTE_ROW_NUM" = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 00050f79d2e6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - {{ quote('DATE') }}, - timestamp_col, - hkd_special___characters, - hkd_special___characters_1, - nzd, - usd, - {{ quote('DATE') }} as {{ quote('_AIRBYTE_START_AT') }}, - lag({{ quote('DATE') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ quote('DATE') }} asc nulls last, {{ quote('DATE') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) as {{ quote('_AIRBYTE_END_AT') }}, - case when lag({{ quote('DATE') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ quote('DATE') }} asc nulls last, {{ quote('DATE') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID') }} -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', 'airbyte_raw_dedup_exchange_rate') }} -where "_AIRBYTE_ROW_NUM" = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index e1e38ef48412..26ed51d88c7b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,23 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + unique_key = "{{ quote('_AIRBYTE_UNIQUE_KEY') }}", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + {{ quote('_AIRBYTE_UNIQUE_KEY') }}, id, name, {{ quote('_AB_CDC_LSN') }}, {{ quote('_AB_CDC_UPDATED_AT') }}, {{ quote('_AB_CDC_DELETED_AT') }}, {{ quote('_AB_CDC_LOG_POS') }}, + {{ quote('_AIRBYTE_AB_ID') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, + {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} -where "_AIRBYTE_ACTIVE_ROW" = 1 +where 1 = 1 +and {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 4abe4a621ad5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - {{ quote('_AB_CDC_LSN') }}, - {{ quote('_AB_CDC_UPDATED_AT') }}, - {{ quote('_AB_CDC_DELETED_AT') }}, - {{ quote('_AB_CDC_LOG_POS') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }} as {{ quote('_AIRBYTE_START_AT') }}, - lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc - ) as {{ quote('_AIRBYTE_END_AT') }}, - case when lag({{ quote('_AIRBYTE_EMITTED_AT') }}) over ( - partition by id - order by {{ quote('_AIRBYTE_EMITTED_AT') }} asc nulls last, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AIRBYTE_EMITTED_AT') }} desc, {{ quote('_AB_CDC_UPDATED_AT') }} desc, {{ quote('_AB_CDC_LOG_POS') }} desc - ) is null and {{ quote('_AB_CDC_DELETED_AT') }} is null then 1 else 0 end as {{ quote('_AIRBYTE_ACTIVE_ROW') }}, - {{ quote('_AIRBYTE_EMITTED_AT') }}, - {{ quote('_AIRBYTE_POS_DEDUP_CDCX_HASHID') }} -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', 'airbyte_raw_pos_dedup_cdcx') }} -where "_AIRBYTE_ROW_NUM" = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql index da67fdfcb24e..f9575722cc41 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -1,10 +1,13 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ 'id', 'name', - 'column___with__quotes', quote('_AB_CDC_LSN'), quote('_AB_CDC_UPDATED_AT'), quote('_AB_CDC_DELETED_AT'), @@ -12,4 +15,6 @@ select tmp.* from {{ ref('dedup_cdc_excluded_ab2') }} tmp -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 64% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 32f16e62fef2..48905c777a2d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +18,6 @@ select tmp.* from {{ ref('dedup_exchange_rate_ab2') }} tmp -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 68% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index bbe9333a3383..52cf027c3895 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -1,4 +1,8 @@ -{{ config(schema="test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -12,4 +16,5 @@ select tmp.* from {{ ref('pos_dedup_cdcx_ab2') }} tmp -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..7555ff5d1efd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,16 @@ +{{ config( + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', quote('_AIRBYTE_AB_ID')), + schema = "test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as {{ quote('_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID') }}, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause(quote('_AIRBYTE_EMITTED_AT')) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml index 954680f83f0e..c6b4b6023ba5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: airbyte_raw_dedup_exchange_rate - name: airbyte_raw_exchange_rate - name: airbyte_raw_pos_dedup_cdcx + - name: airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..bd51b214ebc6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,93 @@ + + + create table test_normalization.dedup_cdc_excluded_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.dedup_cdc_excluded_ab3 + -- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc + ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)) + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_UPDATED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..e6c91980fea5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,99 @@ + + + create table test_normalization.dedup_exchange_rate_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.dedup_exchange_rate_ab3 + -- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id || '~' || + + + currency || '~' || + + + nzd + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "DATE" as "_AIRBYTE_START_AT", + lag("DATE") over ( + partition by id, currency, cast(nzd as varchar2(4000)) + order by + "DATE" asc nulls last, + "DATE" desc, + "_AIRBYTE_EMITTED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("DATE") over ( + partition by id, currency, cast(nzd as varchar2(4000)) + order by + "DATE" asc nulls last, + "DATE" desc, + "_AIRBYTE_EMITTED_AT" desc + ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..c8d9914d17d2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,79 @@ + + + create table test_normalization.renamed_dedup_cdc_excluded_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.renamed_dedup_cdc_excluded_ab3 + -- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc + ) is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT" + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..bf4c8a11c07b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,24 @@ + + + create table test_normalization.dedup_cdc_excluded__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" +from test_normalization.dedup_cdc_excluded_scd +-- dedup_cdc_excluded from test_normalization.airbyte_raw_dedup_cdc_excluded +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..f67f7384cf17 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,27 @@ + + + create table test_normalization.dedup_exchange_rate__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" +from test_normalization.dedup_exchange_rate_scd +-- dedup_exchange_rate from test_normalization.airbyte_raw_dedup_exchange_rate +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..9d77656b65a9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,114 @@ + + + create table test_normalization.exchange_rate__dbt_tmp + + as + +with dbt__cte__exchange_rate_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."currency"') as currency, + json_value("_AIRBYTE_DATA", '$."date"') as "DATE", + json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, + json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, + json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, + json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, + json_value("_AIRBYTE_DATA", '$."USD"') as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_exchange_rate +-- exchange_rate +where 1 = 1 + +), dbt__cte__exchange_rate_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(currency as varchar2(4000)) as currency, + cast(nullif("DATE", '') as + varchar2(4000) +) as "DATE", + cast(nullif(timestamp_col, '') as + varchar2(4000) +) as timestamp_col, + cast(hkd_special___characters as + float +) as hkd_special___characters, + cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__exchange_rate_ab1__ +-- exchange_rate +where 1 = 1 + +), dbt__cte__exchange_rate_ab3__ as ( + +-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + currency || '~' || + + + "DATE" || '~' || + + + timestamp_col || '~' || + + + hkd_special___characters || '~' || + + + hkd_special___characters_1 || '~' || + + + nzd || '~' || + + + usd + + ) as "_AIRBYTE_EXCHANGE_RATE_HASHID", + tmp.* +from dbt__cte__exchange_rate_ab2__ tmp +-- exchange_rate +where 1 = 1 + +)-- Final base SQL model +select + id, + currency, + "DATE", + timestamp_col, + hkd_special___characters, + hkd_special___characters_1, + nzd, + usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_EXCHANGE_RATE_HASHID" +from dbt__cte__exchange_rate_ab3__ +-- exchange_rate from test_normalization.airbyte_raw_exchange_rate +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..feac3368a6ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,20 @@ + + + create table test_normalization.renamed_dedup_cdc_excluded__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" +from test_normalization.renamed_dedup_cdc_excluded_scd +-- renamed_dedup_cdc_excluded from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..c184fd31e2d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,98 @@ + + + create table test_normalization.pos_dedup_cdcx_scd__dbt_tmp + + as + +with + +input_data as ( + select * + from test_normalization.pos_dedup_cdcx_ab3 + -- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + ora_hash( + + id + + ) as "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_EMITTED_AT" as "_AIRBYTE_START_AT", + lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc + ) as "_AIRBYTE_END_AT", + case when lag("_AIRBYTE_EMITTED_AT") over ( + partition by id + order by + "_AIRBYTE_EMITTED_AT" asc nulls last, + "_AIRBYTE_EMITTED_AT" desc, + "_AIRBYTE_EMITTED_AT" desc, "_AB_CDC_UPDATED_AT" desc, "_AB_CDC_LOG_POS" desc + ) is null and "_AB_CDC_DELETED_AT" is null then 1 else 0 end as "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_START_AT", "_AIRBYTE_EMITTED_AT", cast("_AB_CDC_DELETED_AT" as varchar2(4000)), cast("_AB_CDC_UPDATED_AT" as varchar2(4000)), cast("_AB_CDC_LOG_POS" as varchar2(4000)) + order by "_AIRBYTE_AB_ID" + ) as "_AIRBYTE_ROW_NUM", + ora_hash( + + "_AIRBYTE_UNIQUE_KEY" || '~' || + + + "_AIRBYTE_START_AT" || '~' || + + + "_AIRBYTE_EMITTED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_LOG_POS" + + ) as "_AIRBYTE_UNIQUE_KEY_SCD", + scd_data.* + from scd_data +) +select + "_AIRBYTE_UNIQUE_KEY", + "_AIRBYTE_UNIQUE_KEY_SCD", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_START_AT", + "_AIRBYTE_END_AT", + "_AIRBYTE_ACTIVE_ROW", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" +from dedup_data where "_AIRBYTE_ROW_NUM" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..d3fd87ee5576 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,25 @@ + + + create table test_normalization.pos_dedup_cdcx__dbt_tmp + + as + +-- Final base SQL model +select + "_AIRBYTE_UNIQUE_KEY", + id, + name, + "_AB_CDC_LSN", + "_AB_CDC_UPDATED_AT", + "_AB_CDC_DELETED_AT", + "_AB_CDC_LOG_POS", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT", + "_AIRBYTE_POS_DEDUP_CDCX_HASHID" +from test_normalization.pos_dedup_cdcx_scd +-- pos_dedup_cdcx from test_normalization.airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and "_AIRBYTE_ACTIVE_ROW" = 1 \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..30135e3c86ec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,72 @@ + + create view test_normalization.dedup_cdc_excluded_ab3__dbt_tmp as + +with dbt__cte__dedup_cdc_excluded_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."name"') as name, + json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_dedup_cdc_excluded +-- dedup_cdc_excluded +where 1 = 1 + +), dbt__cte__dedup_cdc_excluded_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(name as varchar2(4000)) as name, + cast("_AB_CDC_LSN" as + float +) as "_AB_CDC_LSN", + cast("_AB_CDC_UPDATED_AT" as + float +) as "_AB_CDC_UPDATED_AT", + cast("_AB_CDC_DELETED_AT" as + float +) as "_AB_CDC_DELETED_AT", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__dedup_cdc_excluded_ab1__ +-- dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + name || '~' || + + + "_AB_CDC_LSN" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" + + ) as "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID", + tmp.* +from dbt__cte__dedup_cdc_excluded_ab2__ tmp +-- dedup_cdc_excluded +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..527196251304 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,91 @@ + + create view test_normalization.dedup_exchange_rate_ab3__dbt_tmp as + +with dbt__cte__dedup_exchange_rate_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."currency"') as currency, + json_value("_AIRBYTE_DATA", '$."date"') as "DATE", + json_value("_AIRBYTE_DATA", '$."timestamp_col"') as timestamp_col, + json_value("_AIRBYTE_DATA", '$."HKD@spéçiäl & characters"') as hkd_special___characters, + json_value("_AIRBYTE_DATA", '$."HKD_special___characters"') as hkd_special___characters_1, + json_value("_AIRBYTE_DATA", '$."NZD"') as nzd, + json_value("_AIRBYTE_DATA", '$."USD"') as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_dedup_exchange_rate +-- dedup_exchange_rate +where 1 = 1 + +), dbt__cte__dedup_exchange_rate_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(currency as varchar2(4000)) as currency, + cast(nullif("DATE", '') as + varchar2(4000) +) as "DATE", + cast(nullif(timestamp_col, '') as + varchar2(4000) +) as timestamp_col, + cast(hkd_special___characters as + float +) as hkd_special___characters, + cast(hkd_special___characters_1 as varchar2(4000)) as hkd_special___characters_1, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__dedup_exchange_rate_ab1__ +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + currency || '~' || + + + "DATE" || '~' || + + + timestamp_col || '~' || + + + hkd_special___characters || '~' || + + + hkd_special___characters_1 || '~' || + + + nzd || '~' || + + + usd + + ) as "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID", + tmp.* +from dbt__cte__dedup_exchange_rate_ab2__ tmp +-- dedup_exchange_rate +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..e1a0af83a632 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,76 @@ + + create view test_normalization.pos_dedup_cdcx_ab3__dbt_tmp as + +with dbt__cte__pos_dedup_cdcx_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + json_value("_AIRBYTE_DATA", '$."name"') as name, + json_value("_AIRBYTE_DATA", '$."_ab_cdc_lsn"') as "_AB_CDC_LSN", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_updated_at"') as "_AB_CDC_UPDATED_AT", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_deleted_at"') as "_AB_CDC_DELETED_AT", + json_value("_AIRBYTE_DATA", '$."_ab_cdc_log_pos"') as "_AB_CDC_LOG_POS", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_pos_dedup_cdcx +-- pos_dedup_cdcx +where 1 = 1 +), dbt__cte__pos_dedup_cdcx_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + cast(name as varchar2(4000)) as name, + cast("_AB_CDC_LSN" as + float +) as "_AB_CDC_LSN", + cast("_AB_CDC_UPDATED_AT" as + float +) as "_AB_CDC_UPDATED_AT", + cast("_AB_CDC_DELETED_AT" as + float +) as "_AB_CDC_DELETED_AT", + cast("_AB_CDC_LOG_POS" as + float +) as "_AB_CDC_LOG_POS", + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__pos_dedup_cdcx_ab1__ +-- pos_dedup_cdcx +where 1 = 1 +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id || '~' || + + + name || '~' || + + + "_AB_CDC_LSN" || '~' || + + + "_AB_CDC_UPDATED_AT" || '~' || + + + "_AB_CDC_DELETED_AT" || '~' || + + + "_AB_CDC_LOG_POS" + + ) as "_AIRBYTE_POS_DEDUP_CDCX_HASHID", + tmp.* +from dbt__cte__pos_dedup_cdcx_ab2__ tmp +-- pos_dedup_cdcx +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..6ad67cd02a90 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,46 @@ + + create view test_normalization.renamed_dedup_cdc_excluded_ab3__dbt_tmp as + +with dbt__cte__renamed_dedup_cdc_excluded_ab1__ as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value("_AIRBYTE_DATA", '$."id"') as id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from test_normalization.airbyte_raw_renamed_dedup_cdc_excluded +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), dbt__cte__renamed_dedup_cdc_excluded_ab2__ as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + numeric +) as id, + "_AIRBYTE_AB_ID", + "_AIRBYTE_EMITTED_AT", + + CURRENT_TIMESTAMP + as "_AIRBYTE_NORMALIZED_AT" +from dbt__cte__renamed_dedup_cdc_excluded_ab1__ +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + ora_hash( + + id + + ) as "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID", + tmp.* +from dbt__cte__renamed_dedup_cdc_excluded_ab2__ tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql deleted file mode 100644 index 6748eb12b538..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab4__dbt_tmp" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stre__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3" tmp --- nested_stream_with_c__lting_into_long_names from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql deleted file mode 100644 index 8601af054b1a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - create table "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd__dbt_tmp" - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - "id", - "date", - "partition", - "date" as _airbyte_start_at, - lag("date") over ( - partition by "id" - order by "date" is null asc, "date" desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag("date") over ( - partition by "id" - order by "date" is null asc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stre__nto_long_names_hashid -from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab4" --- nested_stream_with_c__lting_into_long_names from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index 693121b8cdc5..ce2e7a3ec5b5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -5,7 +5,10 @@ select jsonb_extract_path_text(_airbyte_data, 'id') as "id", jsonb_extract_path(_airbyte_data, 'conflict_stream_array') as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization._airbyte_raw_conflict_stream_array as table_alias -- conflict_stream_array +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 7cb90ed2af78..a4df9ebe22b0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -7,7 +7,10 @@ select varchar ) as "id", conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."conflict_stream_array_ab1" -- conflict_stream_array +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 5ba21b8a148f..de392f8c307b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."conflict_stream_array_ab2" tmp -- conflict_stream_array +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql index 6269ba0ab954..da03bc0bb8db 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql @@ -5,8 +5,11 @@ select _airbyte_conflict_stream_name_2_hashid, jsonb_extract_path_text(conflict_stream_name, 'groups') as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql index 671aa9e8df7d..f40efe7753f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql @@ -7,7 +7,10 @@ select cast(groups as varchar ) as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1" -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql index acdc2bce21b1..585ae9060bfe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2" tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 8d5e15e1cd45..edec407e9552 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -7,7 +7,10 @@ select jsonb_extract_path(table_alias._airbyte_data, 'conflict_stream_name') as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization._airbyte_raw_conflict_stream_name as table_alias -- conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index 5bbbb58c1832..fed1290ad9a7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -9,7 +9,10 @@ select cast(conflict_stream_name as jsonb ) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."conflict_stream_name_ab1" -- conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index 4d7cf4c3ae76..d82b4294172d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."conflict_stream_name_ab2" tmp -- conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index af9e0733f509..f0ca905a6656 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -7,8 +7,11 @@ select jsonb_extract_path(table_alias.conflict_stream_name, 'conflict_stream_name') as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization."conflict_stream_name" as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 3df5e1f853fa..a3a5870bec5d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -7,7 +7,10 @@ select cast(conflict_stream_name as jsonb ) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index 255cc446e679..8435a2464b7b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index c6c3552e6f57..abb9d5899986 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -5,7 +5,10 @@ select jsonb_extract_path_text(_airbyte_data, 'id') as "id", jsonb_extract_path_text(_airbyte_data, 'conflict_stream_scalar') as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias -- conflict_stream_scalar +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index 06644717b27d..a5a94a9cb6c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -9,7 +9,10 @@ select cast(conflict_stream_scalar as bigint ) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1" -- conflict_stream_scalar +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 7435231cc854..66ad40a66f43 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp -- conflict_stream_scalar +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql index a15d7c286441..f6e2700418b3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql @@ -7,8 +7,11 @@ select jsonb_extract_path("partition", 'double_array_data') as double_array_data, jsonb_extract_path("partition", 'DATA') as "DATA", jsonb_extract_path("partition", 'column`_''with"_quotes') as "column`_'with""_quotes", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" as table_alias -where "partition" is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql index 8beca5ca9421..451bd82174e7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql @@ -7,7 +7,10 @@ select double_array_data, "DATA", "column`_'with""_quotes", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1" -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql index 0d745ed71a40..eb31fad896ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql @@ -17,4 +17,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2" tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql index 7257a111900c..602dad2547f8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql @@ -6,13 +6,16 @@ select _airbyte_partition_hashid, jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA cross join jsonb_array_elements( case jsonb_typeof("DATA") when 'array' then "DATA" else '[]' end ) as _airbyte_nested_data -where "DATA" is not null --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 +and "DATA" is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql index 190e0bc7efdc..c51fbcb9e61f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql @@ -7,7 +7,10 @@ select cast(currency as varchar ) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1" -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql index 4d934eb2ca2e..94e1500a8681 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2" tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql index 4f5262e747d7..d6e58ff5015a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql @@ -6,13 +6,16 @@ select _airbyte_partition_hashid, jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes cross join jsonb_array_elements( case jsonb_typeof("column`_'with""_quotes") when 'array' then "column`_'with""_quotes" else '[]' end ) as _airbyte_nested_data -where "column`_'with""_quotes" is not null --- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 +and "column`_'with""_quotes" is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql index 6731a64c8070..3517ce920c9a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql @@ -7,7 +7,10 @@ select cast(currency as varchar ) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1" -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql index 46bb87ddebc0..37d739b12c2c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2" tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql index e6cab758b3bd..e18ebddd1a8c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql @@ -6,13 +6,16 @@ select _airbyte_partition_hashid, jsonb_extract_path_text(_airbyte_nested_data, 'id') as "id", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data cross join jsonb_array_elements( case jsonb_typeof(double_array_data) when 'array' then double_array_data else '[]' end ) as _airbyte_nested_data -where double_array_data is not null --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 +and double_array_data is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql index dab5ccb08a17..d716b1eb2f5d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql @@ -7,7 +7,10 @@ select cast("id" as varchar ) as "id", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1" -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql index 161af77ca437..f4511754ecfc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2" tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql index 0cc444ec6bf5..fd8bdeb88d0c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql @@ -8,7 +8,11 @@ select jsonb_extract_path(table_alias._airbyte_data, 'partition') as "partition", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias -- nested_stream_with_c__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql index 90dadf5dd098..3eacd30f282f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql @@ -12,7 +12,11 @@ select cast("partition" as jsonb ) as "partition", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1" -- nested_stream_with_c__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql index 27fd33e91cea..2a1e4ca28f20 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql @@ -5,7 +5,10 @@ select jsonb_extract_path_text(_airbyte_data, 'id') as "id", jsonb_extract_path_text(_airbyte_data, 'date') as "date", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias -- non_nested_stream_wi__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql index 648f2c41da70..51620aee9290 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql @@ -9,7 +9,10 @@ select cast("date" as varchar ) as "date", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1" -- non_nested_stream_wi__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql index 40ba8b06e03d..151cdb6e10a3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2" tmp -- non_nested_stream_wi__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 018a584a1ac0..b70f91add752 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -5,7 +5,11 @@ select jsonb_extract_path_text(_airbyte_data, 'id') as "id", jsonb_extract_path(_airbyte_data, 'children') as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization._airbyte_raw_unnest_alias as table_alias -- unnest_alias +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql similarity index 77% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index 6e420b273a89..86b9fb2c93f3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -7,7 +7,11 @@ select bigint ) as "id", children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."unnest_alias_ab1" -- unnest_alias +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 182ddc64ac42..f23e4c1aca53 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -13,4 +13,6 @@ select tmp.* from "postgres"._airbyte_test_normalization."unnest_alias_ab2" tmp -- unnest_alias +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index c14e14b49980..04ce3aa2eacc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -9,13 +9,16 @@ select jsonb_extract_path(_airbyte_nested_data, 'owner') as "owner", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children cross join jsonb_array_elements( case jsonb_typeof(children) when 'array' then children else '[]' end ) as _airbyte_nested_data -where children is not null --- children at unnest_alias/children +where 1 = 1 +and children is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 75278719675d..00521425f33e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -10,7 +10,10 @@ select cast("owner" as jsonb ) as "owner", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."unnest_alias_children_ab1" -- children at unnest_alias/children +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index 8aa509953c79..f11f03175e3b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -15,4 +15,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."unnest_alias_children_ab2" tmp -- children at unnest_alias/children +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 1d3ff52740e1..3aafcf647131 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -5,8 +5,11 @@ select _airbyte_children_hashid, jsonb_extract_path_text("owner", 'owner_id') as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization."unnest_alias_children" as table_alias -where "owner" is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and "owner" is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index 773916c973af..b2ca5615ca90 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -7,7 +7,10 @@ select cast(owner_id as bigint ) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1" -- owner at unnest_alias/children/owner +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index 4f2ca0385d91..8239aba10289 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp -- owner at unnest_alias/children/owner +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql index c2357389f4d2..5211a6eeff16 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql @@ -5,7 +5,10 @@ select jsonb_extract_path_text(_airbyte_data, 'id') as "id", jsonb_extract_path_text(_airbyte_data, 'date') as "date", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias -- simple_stream_with_n__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql index 9beefb4a2609..8f67edcf78c3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql @@ -9,7 +9,10 @@ select cast("date" as varchar ) as "date", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1" -- simple_stream_with_n__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql index e34dd44daa29..1c204c3383bb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql @@ -13,4 +13,5 @@ select tmp.* from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2" tmp -- simple_stream_with_n__lting_into_long_names +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..d844645b4961 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -0,0 +1,81 @@ + + + + create table "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3" + -- nested_stream_with_c__lting_into_long_names from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + "date", + "partition", + "date" as _airbyte_start_at, + lag("date") over ( + partition by "id" + order by + "date" is null asc, + "date" desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag("date") over ( + partition by "id" + order by + "date" is null asc, + "date" desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stre__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + "date", + "partition", + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_nested_stre__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql similarity index 73% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql index cf0e6ec3c444..a90caa4b0cb6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql @@ -1,16 +1,23 @@ + - create table "postgres".test_normalization."nested_stream_with_c__lting_into_long_names__dbt_tmp" + create table "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" as ( -- Final base SQL model select + _airbyte_unique_key, "id", "date", "partition", + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_nested_stre__nto_long_names_hashid from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" -- nested_stream_with_c__lting_into_long_names from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_active_row = 1 - ); \ No newline at end of file +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql similarity index 64% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql index 546d7d6db6a1..b1be69446486 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -1,14 +1,20 @@ + - create table "postgres".test_normalization."unnest_alias__dbt_tmp" + create table "postgres".test_normalization."unnest_alias" as ( -- Final base SQL model select "id", children, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_unnest_alias_hashid from "postgres"._airbyte_test_normalization."unnest_alias_ab3" -- unnest_alias from "postgres".test_normalization._airbyte_raw_unnest_alias - ); \ No newline at end of file +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql index 82df6d490af6..24d04f946c89 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -7,8 +7,11 @@ select "id", conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from "postgres"._airbyte_test_normalization."conflict_stream_array_ab3" -- conflict_stream_array from "postgres".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql index 2621ce2ba6eb..a664bb56d781 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -7,8 +7,11 @@ select "id", conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from "postgres"._airbyte_test_normalization."conflict_stream_name_ab3" -- conflict_stream_name from "postgres".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql index 42f7228477cb..0e06a3243fa2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql @@ -7,8 +7,11 @@ select _airbyte_conflict_stream_name_2_hashid, groups, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3" -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index d78bcb97cd2a..df6e327e1620 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -7,8 +7,11 @@ select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" -- conflict_stream_name at conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql index a1fa74a3691d..bdcd7a2664c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -7,8 +7,11 @@ select "id", conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3" -- conflict_stream_scalar from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql similarity index 89% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql index 05b5585f6de3..a9781fcf92d2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql @@ -9,8 +9,11 @@ select double_array_data, "DATA", "column`_'with""_quotes", + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_partition_hashid from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3" -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql index 8a5b484f27c8..b6c629700a04 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql @@ -7,8 +7,11 @@ select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_data_hashid from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3" -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql index 357223729efc..037c53598daf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql @@ -7,8 +7,11 @@ select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3" -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql index a1ee4e50790b..6d3ed538c39d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql @@ -7,8 +7,11 @@ select _airbyte_partition_hashid, "id", + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_double_array_data_hashid from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3" -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql index f3d8553a5893..0939d42acba7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql @@ -7,8 +7,11 @@ select "id", "date", + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_non_nested___nto_long_names_hashid from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3" -- non_nested_stream_wi__lting_into_long_names from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql index 36d780ad4cac..5248364bc8cd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -8,8 +8,11 @@ select _airbyte_unnest_alias_hashid, ab_id, "owner", + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_children_hashid from "postgres"._airbyte_test_normalization."unnest_alias_children_ab3" -- children at unnest_alias/children from "postgres".test_normalization."unnest_alias" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 7913f47c46e9..b3074c01b9c9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -7,8 +7,11 @@ select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_owner_hashid from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3" -- owner at unnest_alias/children/owner from "postgres".test_normalization."unnest_alias_children" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql index 87e70a69c3e6..ed8e802bc153 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql @@ -7,8 +7,11 @@ select "id", "date", + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_simple_stre__nto_long_names_hashid from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3" -- simple_stream_with_n__lting_into_long_names from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql index b7e5ed3ee3dc..0d5dd2d7fcac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/final/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql @@ -15,4 +15,6 @@ select tmp.* from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2" tmp -- nested_stream_with_c__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 62d0ec8cc61b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_c__lting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_c___long_names_partition') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('nested_stream_with_c___names_partition_data') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('nested_stream_with_c__ion_double_array_data') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index bcc29f442b1c..b6d1d437f60a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2f109e54445b..44a922d69754 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_array_ab1') }} -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 5422c2c86745..c2ab9a61b17c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_array_ab2') }} tmp -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql index 77e7bd31c512..08d114e6ce78 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_2_hashid, {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql index 683cf66727ac..8179e5d3f565 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast(groups as {{ dbt_utils.type_string() }}) as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name___conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql index 25a48098b5cf..91728d4b7441 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name___conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 1026b2100308..52ad1cb15400 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index 020604d3bd83..19af2a871603 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_ab1') }} -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index 9d6d6c50ccf9..3c048edff728 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name_ab2') }} tmp -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index d6e2edb75ced..62d7cc9c812b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_hashid, {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 463a878933f8..5d564afc1564 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index f8d96123ce25..3e8ebb5d34d3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index 95f08b861d80..87f7dae0e73e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index cd0a70325db6..1f254edd07e1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_scalar_ab1') }} -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index bafebc8d9113..8a1fb1abcdb9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_scalar_ab2') }} tmp -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql index 11d7c57a019f..c23eb81cdb1f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_nested_stre__nto_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c__lting_into_long_names') }} as table_alias -where {{ adapter.quote('partition') }} is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and {{ adapter.quote('partition') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql index d334fa9e3af9..de62103f6c01 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql @@ -1,11 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_nested_stre__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___long_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql index a6fe0d5760ef..03b4ba44eaa8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +15,5 @@ select tmp.* from {{ ref('nested_stream_with_c___long_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql index 871d33d96ae1..5c39f919bd0c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_c___long_names_partition', 'partition', adapter.quote('DATA')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('DATA')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('DATA')) }} -where {{ adapter.quote('DATA') }} is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('partition', adapter.quote('DATA')) }} +where 1 = 1 +and {{ adapter.quote('DATA') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql index 4034a1495a09..670fef499d16 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___names_partition_data_ab1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql index 4e30a677a9e7..962f839574c5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_c___names_partition_data_ab2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql index 31ac76876f60..7445f62f1469 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_c___long_names_partition', 'partition', adapter.quote('column`_\'with""_quotes')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} -where {{ adapter.quote('column`_\'with""_quotes') }} is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column`_\'with""_quotes') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql index 1f1d43c09a26..94780e7b4e93 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c__column___with__quotes_ab1') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql index d6ea901a5e6f..8bf1b1d6c988 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_c__column___with__quotes_ab2') }} tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql index 83232f67277e..d5254e59dd9e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_c___long_names_partition', 'partition', 'double_array_data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as {{ adapter.quote('id') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c___long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'double_array_data') }} -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('partition', 'double_array_data') }} +where 1 = 1 +and double_array_data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql index bcaa8b5c53d5..0f5f7ae3a86e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c__ion_double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql index 351d652df557..7231c119bc08 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_c__ion_double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql index 5888a043ab45..6d69fcd436cf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -- nested_stream_with_c__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql index 11d64cbafcba..a257c6a59026 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_c__lting_into_long_names_ab1') }} -- nested_stream_with_c__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql deleted file mode 100644 index 6a964a409427..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stre__nto_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('nested_stream_with_c__lting_into_long_names_ab3') }} tmp --- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql index b844098ba9c0..9f67e97c0ad1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias -- non_nested_stream_wi__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql index 5b3a3574bf40..054b91557303 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('non_nested_stream_wi__lting_into_long_names_ab1') }} -- non_nested_stream_wi__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql index c8e27a19a68d..1c7fb942b403 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('non_nested_stream_wi__lting_into_long_names_ab2') }} tmp -- non_nested_stream_wi__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index a0e70f6b55be..fbf3b9b1c3a2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index 6fb7cfe48e7b..0af0456e3965 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_ab1') }} -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 340b65b8eeaa..39fdd94517f6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,6 @@ select tmp.* from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 2237b274e29b..45a193100da8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} select _airbyte_unnest_alias_hashid, {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as {{ adapter.quote('owner') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias') }} as table_alias -{{ cross_join_unnest('unnest_alias', 'children') }} -where children is not null -- children at unnest_alias/children +{{ cross_join_unnest('unnest_alias', 'children') }} +where 1 = 1 +and children is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 435447589db2..b93715ce6a95 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_unnest_alias_hashid, cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, cast({{ adapter.quote('owner') }} as {{ type_json() }}) as {{ adapter.quote('owner') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_ab1') }} -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index e1fa4b470cc8..5080e4179aef 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +14,5 @@ select tmp.* from {{ ref('unnest_alias_children_ab2') }} tmp -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 9085180f4824..6ae9fd7cf4ac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_children_hashid, {{ json_extract_scalar(adapter.quote('owner'), ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children') }} as table_alias -where {{ adapter.quote('owner') }} is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and {{ adapter.quote('owner') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index a0f474c62db7..4ca313879bc7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_owner_ab1') }} -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index c5f6562c34c2..a25ec48b8a56 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql index 06a99b3588c8..b78dcd21c316 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias -- simple_stream_with_n__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql index 70077c18dbfc..14ce22350f65 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('id') }}, cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('simple_stream_with_n__lting_into_long_names_ab1') }} -- simple_stream_with_n__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql index 2d932f59e7a9..9d005b773352 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('simple_stream_with_n__lting_into_long_names_ab2') }} tmp -- simple_stream_with_n__lting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..19fc71c02c38 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -0,0 +1,108 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('nested_stream_with_c__lting_into_long_names_ab3') }} + -- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('nested_stream_with_c__lting_into_long_names_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('nested_stream_with_c__lting_into_long_names_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('nested_stream_with_c__lting_into_long_names_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('nested_stream_with_c__lting_into_long_names_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('nested_stream_with_c__lting_into_long_names_ab3') }} + -- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by {{ adapter.quote('id') }} + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by {{ adapter.quote('id') }} + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stre__nto_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_stre__nto_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql similarity index 52% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql index 6cec45f29f02..9b80b49946e7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql @@ -1,12 +1,22 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, {{ adapter.quote('id') }}, {{ adapter.quote('date') }}, {{ adapter.quote('partition') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_nested_stre__nto_long_names_hashid from {{ ref('nested_stream_with_c__lting_into_long_names_scd') }} -- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..2dcfc8cdeb65 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,19 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + {{ adapter.quote('id') }}, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index d174a85b7a46..2bbf1b892334 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select {{ adapter.quote('id') }}, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from {{ ref('conflict_stream_array_ab3') }} -- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 87a6ace1c885..5a93704d86f2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select {{ adapter.quote('id') }}, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from {{ ref('conflict_stream_name_ab3') }} -- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql index 7be580e5b59a..dcf9220bfb32 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, groups, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from {{ ref('conflict_stream_name___conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 94a9dc3d0139..54f7afc91440 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index a36056903767..54c2c46282db 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select {{ adapter.quote('id') }}, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from {{ ref('conflict_stream_scalar_ab3') }} -- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql index 32fac08a6d40..e8cac7be4d93 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql @@ -1,12 +1,20 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_nested_stre__nto_long_names_hashid, double_array_data, {{ adapter.quote('DATA') }}, {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_partition_hashid from {{ ref('nested_stream_with_c___long_names_partition_ab3') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_c__lting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql index 9ba3d4c84f6e..f254f99ad20a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_data_hashid from {{ ref('nested_stream_with_c___names_partition_data_ab3') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_c___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql index c5af87800f26..75c2b74581e6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from {{ ref('nested_stream_with_c__column___with__quotes_ab3') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_c___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql index 18d2b45b021d..9311ea360679 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, {{ adapter.quote('id') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_double_array_data_hashid from {{ ref('nested_stream_with_c__ion_double_array_data_ab3') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_c___long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql deleted file mode 100644 index e32972366642..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - {{ adapter.quote('id') }}, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }} - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }} - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stre__nto_long_names_hashid -from {{ ref('nested_stream_with_c__lting_into_long_names_ab4') }} --- nested_stream_with_c__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql index f9fe1a5fdf4e..cabe1843d8af 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select {{ adapter.quote('id') }}, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_non_nested___nto_long_names_hashid from {{ ref('non_nested_stream_wi__lting_into_long_names_ab3') }} -- non_nested_stream_wi__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index cdd45120be1c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - {{ adapter.quote('id') }}, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 9ce667c8b142..397f6ab07fb5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,11 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, {{ adapter.quote('owner') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_children_hashid from {{ ref('unnest_alias_children_ab3') }} -- children at unnest_alias/children from {{ ref('unnest_alias') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 0ccf8e33d8d6..7aeb294ab08d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_owner_hashid from {{ ref('unnest_alias_children_owner_ab3') }} -- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql index 247047c0ec69..44d939eb5a0d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} -- Final base SQL model select {{ adapter.quote('id') }}, {{ adapter.quote('date') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_simple_stre__nto_long_names_hashid from {{ ref('simple_stream_with_n__lting_into_long_names_ab3') }} -- simple_stream_with_n__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql similarity index 56% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql index 658f253418f7..b0f0198eb0e3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +14,6 @@ select tmp.* from {{ ref('nested_stream_with_c__lting_into_long_names_ab2') }} tmp -- nested_stream_with_c__lting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..ce2e7a3ec5b5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path(_airbyte_data, 'conflict_stream_array') as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..a4df9ebe22b0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_array_ab1" +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..de392f8c307b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_array_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_array as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_array_ab2" tmp +-- conflict_stream_array +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..da03bc0bb8db --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab1.sql @@ -0,0 +1,15 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + jsonb_extract_path_text(conflict_stream_name, 'groups') as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..f40efe7753f1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as + varchar +) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..585ae9060bfe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name___conflict_stream_name_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + varchar +), '') || '-' || coalesce(cast(groups as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..edec407e9552 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + + jsonb_extract_path(table_alias._airbyte_data, 'conflict_stream_name') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..fed1290ad9a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast(conflict_stream_name as + jsonb +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_name_ab1" +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..d82b4294172d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_name as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_name_ab2" tmp +-- conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..f0ca905a6656 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + jsonb_extract_path(table_alias.conflict_stream_name, 'conflict_stream_name') + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..a3a5870bec5d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as + jsonb +) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab1" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..8435a2464b7b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as + varchar +), '') || '-' || coalesce(cast(conflict_stream_name as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab2" tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..abb9d5899986 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'conflict_stream_scalar') as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..a5a94a9cb6c1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab1" +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..66ad40a66f43 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(conflict_stream_scalar as + varchar +), '') as + varchar +)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab2" tmp +-- conflict_stream_scalar +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql new file mode 100644 index 000000000000..f6e2700418b3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab1.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stre__nto_long_names_hashid, + jsonb_extract_path("partition", 'double_array_data') as double_array_data, + jsonb_extract_path("partition", 'DATA') as "DATA", + jsonb_extract_path("partition", 'column`_''with"_quotes') as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql new file mode 100644 index 000000000000..451bd82174e7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stre__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab1" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql new file mode 100644 index 000000000000..eb31fad896ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___long_names_partition_ab3.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_nested_stre__nto_long_names_hashid as + varchar +), '') || '-' || coalesce(cast(double_array_data as + varchar +), '') || '-' || coalesce(cast("DATA" as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _airbyte_partition_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab2" tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql new file mode 100644 index 000000000000..602dad2547f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join jsonb_array_elements( + case jsonb_typeof("DATA") + when 'array' then "DATA" + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and "DATA" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql new file mode 100644 index 000000000000..c51fbcb9e61f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + varchar +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab1" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql new file mode 100644 index 000000000000..94e1500a8681 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c___names_partition_data_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') as + varchar +)) as _airbyte_data_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab2" tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql new file mode 100644 index 000000000000..d6e58ff5015a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'currency') as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +cross join jsonb_array_elements( + case jsonb_typeof("column`_'with""_quotes") + when 'array' then "column`_'with""_quotes" + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and "column`_'with""_quotes" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql new file mode 100644 index 000000000000..3517ce920c9a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + varchar +) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab1" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql new file mode 100644 index 000000000000..37d739b12c2c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__column___with__quotes_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') as + varchar +)) as _airbyte_column___with__quotes_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab2" tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql new file mode 100644 index 000000000000..e18ebddd1a8c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'id') as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."nested_stream_with_c___long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join jsonb_array_elements( + case jsonb_typeof(double_array_data) + when 'array' then double_array_data + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and double_array_data is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql new file mode 100644 index 000000000000..d716b1eb2f5d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast("id" as + varchar +) as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab1" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql new file mode 100644 index 000000000000..f4511754ecfc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__ion_double_array_data_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as + varchar +), '') || '-' || coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_double_array_data_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab2" tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..fd8bdeb88d0c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + + jsonb_extract_path(table_alias._airbyte_data, 'partition') + as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..3eacd30f282f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/nested_stream_with_c__lting_into_long_names_ab2.sql @@ -0,0 +1,22 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + cast("partition" as + jsonb +) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab1" +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..2a1e4ca28f20 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..51620aee9290 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab1" +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..151cdb6e10a3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/non_nested_stream_wi__lting_into_long_names_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') as + varchar +)) as _airbyte_non_nested___nto_long_names_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab2" tmp +-- non_nested_stream_wi__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..b70f91add752 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,15 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path(_airbyte_data, 'children') as children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..86b9fb2c93f3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + children, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."unnest_alias_ab1" +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..f23e4c1aca53 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(children as + varchar +), '') as + varchar +)) as _airbyte_unnest_alias_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."unnest_alias_ab2" tmp +-- unnest_alias +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..04ce3aa2eacc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,24 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + jsonb_extract_path_text(_airbyte_nested_data, 'ab_id') as ab_id, + + jsonb_extract_path(_airbyte_nested_data, 'owner') + as "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children +cross join jsonb_array_elements( + case jsonb_typeof(children) + when 'array' then children + else '[]' end + ) as _airbyte_nested_data +where 1 = 1 +and children is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..00521425f33e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,19 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast("owner" as + jsonb +) as "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."unnest_alias_children_ab1" +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..f11f03175e3b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,19 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as + varchar +), '') || '-' || coalesce(cast(ab_id as + varchar +), '') || '-' || coalesce(cast("owner" as + varchar +), '') as + varchar +)) as _airbyte_children_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."unnest_alias_children_ab2" tmp +-- children at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..3aafcf647131 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,15 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + jsonb_extract_path_text("owner", 'owner_id') as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and "owner" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..b2ca5615ca90 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab1" +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..8239aba10289 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_children_hashid as + varchar +), '') || '-' || coalesce(cast(owner_id as + varchar +), '') as + varchar +)) as _airbyte_owner_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab2" tmp +-- owner at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..5211a6eeff16 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..8f67edcf78c3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab2.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + varchar +) as "id", + cast("date" as + varchar +) as "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab1" +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..1c204c3383bb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_ctes/test_normalization_namespace/simple_stream_with_n__lting_into_long_names_ab3.sql @@ -0,0 +1,17 @@ + + create view "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') as + varchar +)) as _airbyte_simple_stre__nto_long_names_hashid, + tmp.* +from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab2" tmp +-- simple_stream_with_n__lting_into_long_names +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..327e14609a57 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "nested_stream_with_c__lting_into_long_name__dbt_tmp" + ); + + insert into "postgres".test_normalization."nested_stream_with_c__lting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stre__nto_long_names_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stre__nto_long_names_hashid" + from "nested_stream_with_c__lting_into_long_name__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql new file mode 100644 index 000000000000..384d4a63247d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_c__lting_into_long_names.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "nested_stream_with_c__lting_into_long_name__dbt_tmp" + ); + + insert into "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stre__nto_long_names_hashid") + ( + select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stre__nto_long_names_hashid" + from "nested_stream_with_c__lting_into_long_name__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..7199e9f5f914 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."unnest_alias" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "unnest_alias__dbt_tmp" + ); + + insert into "postgres".test_normalization."unnest_alias" ("id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid") + ( + select "id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid" + from "unnest_alias__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..24d04f946c89 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_array__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_array_ab3" +-- conflict_stream_array from "postgres".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..a664bb56d781 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_name__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_name_ab3" +-- conflict_stream_name from "postgres".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql new file mode 100644 index 000000000000..0e06a3243fa2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_name___conflict_stream_name__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_name___conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..df6e327e1620 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_name_conflict_stream_name_ab3" +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from "postgres".test_normalization."conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..bdcd7a2664c1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."conflict_stream_scalar__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from "postgres"._airbyte_test_normalization."conflict_stream_scalar_ab3" +-- conflict_stream_scalar from "postgres".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql new file mode 100644 index 000000000000..a9781fcf92d2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___long_names_partition.sql @@ -0,0 +1,19 @@ + + + create table "postgres".test_normalization."nested_stream_with_c___long_names_partition__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_nested_stre__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_partition_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c___long_names_partition_ab3" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "postgres".test_normalization."nested_stream_with_c__lting_into_long_names" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql new file mode 100644 index 000000000000..b6c629700a04 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c___names_partition_data.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."nested_stream_with_c___names_partition_data__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_data_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c___names_partition_data_ab3" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql new file mode 100644 index 000000000000..037c53598daf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__column___with__quotes.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."nested_stream_with_c__column___with__quotes__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c__column___with__quotes_ab3" +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql new file mode 100644 index 000000000000..6d3ed538c39d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_c__ion_double_array_data.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."nested_stream_with_c__ion_double_array_data__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_partition_hashid, + "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from "postgres"._airbyte_test_normalization."nested_stream_with_c__ion_double_array_data_ab3" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "postgres".test_normalization."nested_stream_with_c___long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql new file mode 100644 index 000000000000..0939d42acba7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."non_nested_stream_wi__lting_into_long_names__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_non_nested___nto_long_names_hashid +from "postgres"._airbyte_test_normalization."non_nested_stream_wi__lting_into_long_names_ab3" +-- non_nested_stream_wi__lting_into_long_names from "postgres".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..5248364bc8cd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,18 @@ + + + create table "postgres".test_normalization."unnest_alias_children__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + "owner", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_children_hashid +from "postgres"._airbyte_test_normalization."unnest_alias_children_ab3" +-- children at unnest_alias/children from "postgres".test_normalization."unnest_alias" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..b3074c01b9c9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization."unnest_alias_children_owner__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_owner_hashid +from "postgres"._airbyte_test_normalization."unnest_alias_children_owner_ab3" +-- owner at unnest_alias/children/owner from "postgres".test_normalization."unnest_alias_children" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql new file mode 100644 index 000000000000..ed8e802bc153 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_n__lting_into_long_names.sql @@ -0,0 +1,17 @@ + + + create table "postgres".test_normalization_namespace."simple_stream_with_n__lting_into_long_names__dbt_tmp" + as ( + +-- Final base SQL model +select + "id", + "date", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_simple_stre__nto_long_names_hashid +from "postgres"._airbyte_test_normalization_namespace."simple_stream_with_n__lting_into_long_names_ab3" +-- simple_stream_with_n__lting_into_long_names from "postgres".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..0d5dd2d7fcac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_c__lting_into_long_names_ab3.sql @@ -0,0 +1,20 @@ + + create view "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast("partition" as + varchar +), '') as + varchar +)) as _airbyte_nested_stre__nto_long_names_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."nested_stream_with_c__lting_into_long_names_ab2" tmp +-- nested_stream_with_c__lting_into_long_names +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index f93e05005947..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab4__dbt_tmp" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3" tmp --- dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index 3b69eecda1b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab4__dbt_tmp" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3" tmp --- dedup_exchange_rate from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 5a9fbb127a5b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab4__dbt_tmp" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" tmp --- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index c7e2d80f646c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,28 +0,0 @@ - - - create table "postgres".test_normalization."dedup_cdc_excluded_scd__dbt_tmp" - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - "id", - "name", - "column`_'with""_quotes", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by "id" - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by "id" - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab4" --- dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 3ba356ddbd8a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,34 +0,0 @@ - - - create table "postgres".test_normalization."dedup_exchange_rate_scd__dbt_tmp" - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - "id", - currency, - "date", - timestamp_col, - "HKD@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - "date" as _airbyte_start_at, - lag("date") over ( - partition by "id", currency, cast(nzd as - varchar -) - order by "date" is null asc, "date" desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag("date") over ( - partition by "id", currency, cast(nzd as - varchar -) - order by "date" is null asc, "date" desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab4" --- dedup_exchange_rate from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 5147b9e4d0a7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,28 +0,0 @@ - - - create table "postgres".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - as ( - --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - "id", - "name", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by "id" - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by "id" - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab4" --- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index a0ee7b628ad3..6e1c220c3cbc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -5,11 +5,14 @@ select jsonb_extract_path_text(_airbyte_data, 'id') as "id", jsonb_extract_path_text(_airbyte_data, 'name') as "name", - jsonb_extract_path_text(_airbyte_data, 'column`_''with"_quotes') as "column`_'with""_quotes", jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias -- dedup_cdc_excluded +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index a65dbf0f5351..88c28f9fa253 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -9,9 +9,6 @@ select cast("name" as varchar ) as "name", - cast("column`_'with""_quotes" as - varchar -) as "column`_'with""_quotes", cast(_ab_cdc_lsn as float ) as _ab_cdc_lsn, @@ -21,7 +18,11 @@ select cast(_ab_cdc_deleted_at as float ) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1" -- dedup_cdc_excluded +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql similarity index 90% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 46e7afe4f771..ed7c144f7d77 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -11,7 +11,11 @@ select jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, jsonb_extract_path_text(_airbyte_data, 'USD') as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql similarity index 89% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 52cbfdd9fae4..7e00f6385af7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -27,7 +27,11 @@ select cast(usd as float ) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1" -- dedup_exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql similarity index 90% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index a9de573688ab..6e6073e308e0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -11,7 +11,11 @@ select jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, jsonb_extract_path_text(_airbyte_data, 'USD') as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias -- exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index bcc55df18e53..6cbcedd760a2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -27,7 +27,11 @@ select cast(usd as float ) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."exchange_rate_ab1" -- exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index efc1e74882a1..e2b7bfd34d06 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -25,4 +25,6 @@ select tmp.* from "postgres"._airbyte_test_normalization."exchange_rate_ab2" tmp -- exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql similarity index 89% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index f0ad78951449..7c1fca4961ff 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -9,7 +9,10 @@ select jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, jsonb_extract_path_text(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias -- pos_dedup_cdcx +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql similarity index 86% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index 48cdc7668ca2..a9e2b29e6210 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -21,7 +21,10 @@ select cast(_ab_cdc_log_pos as float ) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1" -- pos_dedup_cdcx +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..4380fdbedd6b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..7371209e09a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..0f6ee557356e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,93 @@ + + + + create table "postgres".test_normalization."dedup_cdc_excluded_scd" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3" + -- dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + varchar +), cast(_ab_cdc_updated_at as + varchar +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..68c9ff694078 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,77 @@ + + + + create table "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" + -- renamed_dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql similarity index 74% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index e4aea1d62935..f19c2bcfd939 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -1,19 +1,25 @@ + - create table "postgres".test_normalization."dedup_cdc_excluded__dbt_tmp" + create table "postgres".test_normalization."dedup_cdc_excluded" as ( -- Final base SQL model select + _airbyte_unique_key, "id", "name", - "column`_'with""_quotes", _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_dedup_cdc_excluded_hashid from "postgres".test_normalization."dedup_cdc_excluded_scd" -- dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_active_row = 1 - ); \ No newline at end of file +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 996d92274da2..9fbeb8463fe0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,10 +1,12 @@ + - create table "postgres".test_normalization."dedup_exchange_rate__dbt_tmp" + create table "postgres".test_normalization."dedup_exchange_rate" as ( -- Final base SQL model select + _airbyte_unique_key, "id", currency, "date", @@ -13,9 +15,14 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from "postgres".test_normalization."dedup_exchange_rate_scd" -- dedup_exchange_rate from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_active_row = 1 - ); \ No newline at end of file +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql index b0a2937dfba2..2c7e88caddef 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -16,6 +16,7 @@ select usd, _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from "postgres"._airbyte_test_normalization."exchange_rate_ab3" -- exchange_rate from "postgres".test_normalization._airbyte_raw_exchange_rate diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..370ce4c75de2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,21 @@ + + + + create table "postgres".test_normalization."renamed_dedup_cdc_excluded" + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" +-- renamed_dedup_cdc_excluded from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..48e2e8023174 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,97 @@ + + + create table "postgres".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + varchar +), cast(_ab_cdc_updated_at as + varchar +), cast(_ab_cdc_log_pos as + varchar +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_log_pos as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql similarity index 77% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index 68057cf35c7f..25e9cb949467 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -5,15 +5,19 @@ -- Final base SQL model select + _airbyte_unique_key, "id", "name", _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + now() as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from "postgres".test_normalization."pos_dedup_cdcx_scd" -- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql similarity index 89% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql index 96bac38f7781..63d83886e5f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -7,8 +7,6 @@ select varchar ), '') || '-' || coalesce(cast("name" as varchar -), '') || '-' || coalesce(cast("column`_'with""_quotes" as - varchar ), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar ), '') || '-' || coalesce(cast(_ab_cdc_updated_at as @@ -21,4 +19,6 @@ select tmp.* from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp -- dedup_cdc_excluded +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index c249b790827c..162849517206 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -25,4 +25,6 @@ select tmp.* from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp -- dedup_exchange_rate +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index 784d9eca5762..fceb899227d1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/final/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -21,4 +21,5 @@ select tmp.* from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp -- pos_dedup_cdcx +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..732f2bd1a1ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 6144a13617e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 4d6554576aca..6ff5c5605fb8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as {{ adapter.quote('name') }}, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index e7941a7141f5..a90b1c1e09af 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, cast({{ adapter.quote('name') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('name') }}, - cast({{ adapter.quote('column`_\'with""_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column`_\'with""_quotes') }}, cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index aa34179c3c5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 2703c06376ce..5575218fafd2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, @@ -9,7 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 9b23934d2e81..74859161cd8f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, @@ -9,7 +14,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index ef5f6e52d60e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index afce0ec584db..e0f71f0e7cf2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,6 @@ {{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -13,8 +15,10 @@ select {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, _airbyte_ab_id, - _airbyte_emitted_at + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 178badcb2596..56f31c5d75f3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,6 @@ {{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -13,8 +15,10 @@ select cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, _airbyte_ab_id, - _airbyte_emitted_at + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index 0469a220171c..8784ce3bc62d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,6 @@ {{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), schema = "_airbyte_test_normalization", tags = [ "top-level-intermediate" ] ) }} @@ -18,4 +20,5 @@ select from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 436a7f04cdaa..fb9d0b0df354 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, @@ -7,7 +12,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index 3a27504a4c19..adf58230d903 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, @@ -7,7 +12,10 @@ select cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 7c897124b395..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..ca844abec811 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,17 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..e2b60bbe8a71 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,17 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast({{ adapter.quote('id') }} as {{ dbt_utils.type_bigint() }}) as {{ adapter.quote('id') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..4e4500b5ad39 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,112 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index ff9e861e971d..44047d79d0cd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,38 +1,122 @@ {{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", schema = "test_normalization", - unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), tags = [ "top-level" ] ) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + {{ adapter.quote('date') }} is null asc, + {{ adapter.quote('date') }} desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) select - {{ dbt_utils.surrogate_key([ - adapter.quote('id'), - 'currency', - 'nzd', - ]) }} as _airbyte_unique_key, - {{ adapter.quote('id') }}, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where 1 = 1 -and _airbyte_row_num = 1 -{{ incremental_clause('_airbyte_emitted_at') }} + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..8c57c2ee55b5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,104 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..18247c346368 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,24 @@ +{{ config( + indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from {{ ref('dedup_cdc_excluded_scd') }} +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index f31846c24b9c..370919ff628e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,6 +1,7 @@ {{ config( - schema = "test_normalization", + indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], unique_key = "_airbyte_unique_key", + schema = "test_normalization", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -16,6 +17,7 @@ select usd, _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from {{ ref('dedup_exchange_rate_scd') }} -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql index 15a9bcb2ac23..b8b52c6d4d88 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -1,6 +1,7 @@ {{ config( - schema = "test_normalization", + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -15,6 +16,7 @@ select usd, _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from {{ ref('exchange_rate_ab3') }} -- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..4bf1ea58e96e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,20 @@ +{{ config( + indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + {{ adapter.quote('id') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..afc9f924fa86 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,114 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by {{ adapter.quote('id') }} + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + {{ adapter.quote('name') }}, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql deleted file mode 100644 index 8faf5847dc66..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_scd') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 3924333f7df3..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by {{ adapter.quote('id') }} - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by {{ adapter.quote('id') }} - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index a1edca0ac0b8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,17 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - {{ adapter.quote('id') }}, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 465a1d012f31..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - {{ adapter.quote('id') }}, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag({{ adapter.quote('date') }}) over ( - partition by {{ adapter.quote('id') }}, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by {{ adapter.quote('date') }} is null asc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index b357729508d7..04c21e5fc3be 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,24 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + indexes = [{'columns':['_airbyte_unique_key','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, {{ adapter.quote('id') }}, {{ adapter.quote('name') }}, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 6226de08b2c0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - {{ adapter.quote('id') }}, - {{ adapter.quote('name') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by {{ adapter.quote('id') }} - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by {{ adapter.quote('id') }} - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql similarity index 55% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql index 6a9767cf21d6..e539d1844de2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -1,10 +1,14 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ adapter.quote('id'), adapter.quote('name'), - adapter.quote('column`_\'with""_quotes'), '_ab_cdc_lsn', '_ab_cdc_updated_at', '_ab_cdc_deleted_at', @@ -12,4 +16,6 @@ select tmp.* from {{ ref('dedup_cdc_excluded_ab2') }} tmp -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 60% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 14b695bdbb6f..ee7631c2c6ea 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +19,6 @@ select tmp.* from {{ ref('dedup_exchange_rate_ab2') }} tmp -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index c3dd502cea84..9da3c6fea93a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -12,4 +17,5 @@ select tmp.* from {{ ref('pos_dedup_cdcx_ab2') }} tmp -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..b152709f2db5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,17 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'hash'}], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml index c085ca8ccc25..45c338b893ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _airbyte_raw_dedup_exchange_rate - name: _airbyte_raw_exchange_rate - name: _airbyte_raw_pos_dedup_cdcx + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..6e1c220c3cbc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..88c28f9fa253 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ + + create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab1" +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..ed7c144f7d77 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..7e00f6385af7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,37 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast(currency as + varchar +) as currency, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + varchar +) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1" +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..6e6073e308e0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'HKD_special___characters') as hkd_special___characters, + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..6cbcedd760a2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,37 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast(currency as + varchar +) as currency, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + varchar +) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."exchange_rate_ab1" +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..e2b7bfd34d06 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(hkd_special___characters as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_exchange_rate_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."exchange_rate_ab2" tmp +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql new file mode 100644 index 000000000000..7c1fca4961ff --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_log_pos') as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql new file mode 100644 index 000000000000..a9e2b29e6210 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab1" +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..4380fdbedd6b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,14 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..7371209e09a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..ff39afab32ca --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "postgres".test_normalization."dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..5536e95b3075 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..c905b883e251 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_cdc_excluded__dbt_tmp" + ); + + insert into "postgres".test_normalization."dedup_cdc_excluded" ("_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..cc0c70d79775 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."dedup_exchange_rate" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_exchange_rate__dbt_tmp" + ); + + insert into "postgres".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + ( + select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from "dedup_exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql index 49f01c196e0f..4e20aa65740e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -6,9 +6,9 @@ from "exchange_rate__dbt_tmp" ); - insert into "postgres".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_exchange_rate_hashid") + insert into "postgres".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") ( - select "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_exchange_rate_hashid" + select "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid" from "exchange_rate__dbt_tmp" ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..a5cc40567b2c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."renamed_dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..48e2e8023174 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,97 @@ + + + create table "postgres".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + as ( + +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by "id" + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as + varchar +), cast(_ab_cdc_updated_at as + varchar +), cast(_ab_cdc_log_pos as + varchar +) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as + varchar +), '') || '-' || coalesce(cast(_airbyte_start_at as + varchar +), '') || '-' || coalesce(cast(_airbyte_emitted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_log_pos as + varchar +), '') as + varchar +)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..25e9cb949467 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,23 @@ + + + create table "postgres".test_normalization."pos_dedup_cdcx__dbt_tmp" + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + "id", + "name", + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from "postgres".test_normalization."pos_dedup_cdcx_scd" +-- pos_dedup_cdcx from "postgres".test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..63d83886e5f1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,24 @@ + + create view "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') as + varchar +)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."dedup_cdc_excluded_ab2" tmp +-- dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..162849517206 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(hkd_special___characters as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..fceb899227d1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,25 @@ + + create view "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_log_pos as + varchar +), '') as + varchar +)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."pos_dedup_cdcx_ab2" tmp +-- pos_dedup_cdcx +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..732f2bd1a1ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,16 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') as + varchar +)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..5fc6f0070232 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'new_column') as new_column, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..1f848d1a326a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,37 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + float +) as "id", + cast(currency as + varchar +) as currency, + cast(new_column as + float +) as new_column, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(nzd as + float +) as nzd, + cast(usd as + bigint +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab1" +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..53eb0e732149 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,21 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'currency') as currency, + jsonb_extract_path_text(_airbyte_data, 'new_column') as new_column, + jsonb_extract_path_text(_airbyte_data, 'date') as "date", + jsonb_extract_path_text(_airbyte_data, 'timestamp_col') as timestamp_col, + jsonb_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters", + jsonb_extract_path_text(_airbyte_data, 'NZD') as nzd, + jsonb_extract_path_text(_airbyte_data, 'USD') as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..6b8c0fa4997e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,37 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + float +) as "id", + cast(currency as + varchar +) as currency, + cast(new_column as + float +) as new_column, + cast(nullif("date", '') as + date +) as "date", + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."exchange_rate_ab1" +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..da83934e7c93 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."exchange_rate_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast(new_column as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_exchange_rate_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."exchange_rate_ab2" tmp +-- exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..b3923b59c83c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,18 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1__dbt_tmp" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'name') as "name", + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_lsn') as _ab_cdc_lsn, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_updated_at') as _ab_cdc_updated_at, + jsonb_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at') as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..aa3cc798fcf9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,28 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2__dbt_tmp" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast("id" as + bigint +) as "id", + cast("name" as + varchar +) as "name", + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab1" +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..7693af7ef2e6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..ecc81c8883b0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."dedup_exchange_rate" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_exchange_rate__dbt_tmp" + ); + + insert into "postgres".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id") + ( + select "_airbyte_unique_key", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id" + from "dedup_exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..381151b8af8c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."exchange_rate" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "exchange_rate__dbt_tmp" + ); + + insert into "postgres".test_normalization."exchange_rate" ("currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id") + ( + select "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id" + from "exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..c8edd1056dd5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "postgres".test_normalization."renamed_dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + + insert into "postgres".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + ( + select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..21806fa208d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,30 @@ + + create view "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast(currency as + varchar +), '') || '-' || coalesce(cast(new_column as + varchar +), '') || '-' || coalesce(cast("date" as + varchar +), '') || '-' || coalesce(cast(timestamp_col as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(nzd as + varchar +), '') || '-' || coalesce(cast(usd as + varchar +), '') as + varchar +)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."dedup_exchange_rate_ab2" tmp +-- dedup_exchange_rate +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..b7b5f10b73a7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,24 @@ + + create view "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast("id" as + varchar +), '') || '-' || coalesce(cast("name" as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_lsn as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_updated_at as + varchar +), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as + varchar +), '') as + varchar +)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from "postgres"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab2" tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql deleted file mode 100644 index 2ff25efaa5dc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ /dev/null @@ -1,69 +0,0 @@ - - - create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd__dbt_tmp" - - - as ( - -with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - - case when json_extract_path_text(table_alias._airbyte_data, 'partition', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'partition', true) end - as "partition", - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias --- nested_stream_with_complex_columns_resulting_into_long_names -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as varchar) as id, - cast(date as varchar) as date, - cast("partition" as varchar) as "partition", - _airbyte_emitted_at -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 --- nested_stream_with_complex_columns_resulting_into_long_names -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast("partition" as varchar), '') as varchar)) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 tmp --- nested_stream_with_complex_columns_resulting_into_long_names -), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab4 as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab3 tmp --- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - date, - "partition", - date as _airbyte_start_at, - lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab4 --- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..c9bc7641794f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,73 @@ + + + + create table + "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_ab3" + -- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, + id, + date, + "partition", + date as _airbyte_start_at, + lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + date, + "partition", + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql index f83a6b593845..0cb6fda50a97 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -1,19 +1,27 @@ + create table - "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names__dbt_tmp" + "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names__dbt_tmp" + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) as ( -- Final base SQL model select + _airbyte_unique_key, id, date, "partition", + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" -- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where _airbyte_active_row = 1 - ); \ No newline at end of file +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql index e11bd8416763..e6dd29796238 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -1,9 +1,11 @@ + create table - "integrationtests".test_normalization."unnest_alias__dbt_tmp" + "integrationtests"."test_normalization"."unnest_alias__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__unnest_alias_ab1 as ( @@ -12,9 +14,13 @@ with __dbt__cte__unnest_alias_ab1 as ( select case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, json_extract_path_text(_airbyte_data, 'children', true) as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization._airbyte_raw_unnest_alias as table_alias -- unnest_alias +where 1 = 1 + ), __dbt__cte__unnest_alias_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -23,9 +29,13 @@ select bigint ) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__unnest_alias_ab1 -- unnest_alias +where 1 = 1 + ), __dbt__cte__unnest_alias_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -34,12 +44,19 @@ select tmp.* from __dbt__cte__unnest_alias_ab2 tmp -- unnest_alias +where 1 = 1 + )-- Final base SQL model select id, children, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_unnest_alias_hashid from __dbt__cte__unnest_alias_ab3 -- unnest_alias from "integrationtests".test_normalization._airbyte_raw_unnest_alias - ); \ No newline at end of file +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql index fba34de13a3c..6a49f0ff964e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."conflict_stream_array__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__conflict_stream_array_ab1 as ( @@ -12,18 +13,24 @@ with __dbt__cte__conflict_stream_array_ab1 as ( select case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, json_extract_path_text(_airbyte_data, 'conflict_stream_array', true) as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array as table_alias -- conflict_stream_array +where 1 = 1 ), __dbt__cte__conflict_stream_array_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as varchar) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__conflict_stream_array_ab1 -- conflict_stream_array +where 1 = 1 ), __dbt__cte__conflict_stream_array_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -32,12 +39,16 @@ select tmp.* from __dbt__cte__conflict_stream_array_ab2 tmp -- conflict_stream_array +where 1 = 1 )-- Final base SQL model select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from __dbt__cte__conflict_stream_array_ab3 -- conflict_stream_array from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql index d23691bcb14a..13aba9129a52 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."conflict_stream_name__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__conflict_stream_name_ab1 as ( @@ -14,18 +15,24 @@ select case when json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) end as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name as table_alias -- conflict_stream_name +where 1 = 1 ), __dbt__cte__conflict_stream_name_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as varchar) as id, cast(conflict_stream_name as varchar) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__conflict_stream_name_ab1 -- conflict_stream_name +where 1 = 1 ), __dbt__cte__conflict_stream_name_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -34,12 +41,16 @@ select tmp.* from __dbt__cte__conflict_stream_name_ab2 tmp -- conflict_stream_name +where 1 = 1 )-- Final base SQL model select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from __dbt__cte__conflict_stream_name_ab3 -- conflict_stream_name from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index e6ee9e646b1f..bab7f7d138d5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 as ( @@ -14,19 +15,25 @@ select case when json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) end as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization."conflict_stream_name" as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null ), __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as varchar) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 ), __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -35,12 +42,16 @@ select tmp.* from __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 )-- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 -- conflict_stream_name at conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql index d49f154857f6..19ac73995aa9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name_conflict_stream_name__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 as ( @@ -12,19 +13,25 @@ with __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ select _airbyte_conflict_stream_name_2_hashid, case when json_extract_path_text(conflict_stream_name, 'groups', true) != '' then json_extract_path_text(conflict_stream_name, 'groups', true) end as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null ), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast(groups as varchar) as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 ), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -33,12 +40,16 @@ select tmp.* from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 )-- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, groups, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 9c4da3e16814..bd6a7edaa276 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."conflict_stream_scalar__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__conflict_stream_scalar_ab1 as ( @@ -12,9 +13,12 @@ with __dbt__cte__conflict_stream_scalar_ab1 as ( select case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, case when json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) != '' then json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) end as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias -- conflict_stream_scalar +where 1 = 1 ), __dbt__cte__conflict_stream_scalar_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -23,9 +27,12 @@ select cast(conflict_stream_scalar as bigint ) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__conflict_stream_scalar_ab1 -- conflict_stream_scalar +where 1 = 1 ), __dbt__cte__conflict_stream_scalar_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -34,12 +41,16 @@ select tmp.* from __dbt__cte__conflict_stream_scalar_ab2 tmp -- conflict_stream_scalar +where 1 = 1 )-- Final base SQL model select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from __dbt__cte__conflict_stream_scalar_ab3 -- conflict_stream_scalar from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index 13de69326ca0..f691cc5da8fb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( @@ -14,10 +15,13 @@ select json_extract_path_text("partition", 'double_array_data', true) as double_array_data, json_extract_path_text("partition", 'DATA', true) as data, json_extract_path_text("partition", 'column`_''with"_quotes', true) as "column`_'with""_quotes", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" as table_alias -where "partition" is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null ), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -26,9 +30,12 @@ select double_array_data, data, "column`_'with""_quotes", - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 ), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -37,14 +44,18 @@ select tmp.* from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 )-- Final base SQL model select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, double_array_data, data, "column`_'with""_quotes", + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_partition_hashid from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql similarity index 90% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql index 2c218b388b5d..9eb1e109077e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 as ( @@ -56,20 +57,26 @@ joined as ( select _airbyte_partition_hashid, case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where "column`_'with""_quotes" is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and "column`_'with""_quotes" is not null ), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as varchar) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 ), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -78,12 +85,16 @@ select tmp.* from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 )-- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql similarity index 90% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql index 50b9df98fc43..ed667f0e4087 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_data__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as ( @@ -56,20 +57,26 @@ joined as ( select _airbyte_partition_hashid, case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where data is not null -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and data is not null ), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as varchar) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 ), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -78,12 +85,16 @@ select tmp.* from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 tmp -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 )-- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_data_hashid from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql similarity index 90% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index 6c3f2811c690..51f7dcf22ca2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( @@ -56,20 +57,26 @@ joined as ( select _airbyte_partition_hashid, case when json_extract_path_text(_airbyte_nested_data, 'id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'id', true) end as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias -left join joined on _airbyte_partition_hashid = joined._airbyte_hashid -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null ), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(id as varchar) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 ), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -78,12 +85,16 @@ select tmp.* from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 )-- Final base SQL model select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_double_array_data_hashid from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql index 94087ad12a9c..95f350d3abdd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."non_nested_stream_without_namespace_resulting_into_long_names__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 as ( @@ -12,18 +13,24 @@ with __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_a select case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 ), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as varchar) as id, cast(date as varchar) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 ), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -32,12 +39,16 @@ select tmp.* from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 tmp -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 )-- Final base SQL model select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 -- non_nested_stream_without_namespace_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql index 8bc518e99add..aca1269c8d5f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."unnest_alias_children__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__unnest_alias_children_ab1 as ( @@ -59,11 +60,14 @@ select case when json_extract_path_text(_airbyte_nested_data, 'owner', true) != '' then json_extract_path_text(_airbyte_nested_data, 'owner', true) end as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization."unnest_alias" as table_alias -left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid -where children is not null -- children at unnest_alias/children +left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid +where 1 = 1 +and children is not null ), __dbt__cte__unnest_alias_children_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -73,9 +77,12 @@ select bigint ) as ab_id, cast(owner as varchar) as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__unnest_alias_children_ab1 -- children at unnest_alias/children +where 1 = 1 ), __dbt__cte__unnest_alias_children_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -84,13 +91,17 @@ select tmp.* from __dbt__cte__unnest_alias_children_ab2 tmp -- children at unnest_alias/children +where 1 = 1 )-- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, owner, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_children_hashid from __dbt__cte__unnest_alias_children_ab3 -- children at unnest_alias/children from "integrationtests".test_normalization."unnest_alias" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index b385cbabb620..d6c293674dbe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization."unnest_alias_children_owner__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__unnest_alias_children_owner_ab1 as ( @@ -12,10 +13,13 @@ with __dbt__cte__unnest_alias_children_owner_ab1 as ( select _airbyte_children_hashid, case when json_extract_path_text(owner, 'owner_id', true) != '' then json_extract_path_text(owner, 'owner_id', true) end as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization."unnest_alias_children" as table_alias -where owner is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null ), __dbt__cte__unnest_alias_children_owner_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -24,9 +28,12 @@ select cast(owner_id as bigint ) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__unnest_alias_children_owner_ab1 -- owner at unnest_alias/children/owner +where 1 = 1 ), __dbt__cte__unnest_alias_children_owner_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -35,12 +42,16 @@ select tmp.* from __dbt__cte__unnest_alias_children_owner_ab2 tmp -- owner at unnest_alias/children/owner +where 1 = 1 )-- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_owner_hashid from __dbt__cte__unnest_alias_children_owner_ab3 -- owner at unnest_alias/children/owner from "integrationtests".test_normalization."unnest_alias_children" +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql similarity index 86% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql index 6627c7dcc0a1..69dff6931661 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -4,6 +4,7 @@ "integrationtests".test_normalization_namespace."simple_stream_with_namespace_resulting_into_long_names__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 as ( @@ -12,18 +13,24 @@ with __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 as ( select case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 ), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as varchar) as id, cast(date as varchar) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 ), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -32,12 +39,16 @@ select tmp.* from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 tmp -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 )-- Final base SQL model select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 -- simple_stream_with_namespace_resulting_into_long_names from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..361919a65bc2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -0,0 +1,43 @@ + + + create view "integrationtests"._airbyte_test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_ab3__dbt_tmp" as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + + case when json_extract_path_text(table_alias._airbyte_data, 'partition', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'partition', true) end + as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(date as varchar) as date, + cast("partition" as varchar) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast("partition" as varchar), '') as varchar)) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 tmp +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index f08db80c0559..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql index d3df56ab3d35..8338f7a9440b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql index 2f0da1bb06e7..47b4d187fc92 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, conflict_stream_array, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_array_ab1') }} -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql index 0a92c655f92a..d4efd628548e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_array_ab2') }} tmp -- conflict_stream_array +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql index 8556d7f50afb..223381444e1f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql index a1d77f379bce..02a958942c9c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_ab1') }} -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql index dc562af89ba1..89a2bdb2f1ed 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name_ab2') }} tmp -- conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql index d6e2edb75ced..a8a342548c36 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_hashid, {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql index 463a878933f8..3c458a14224f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_hashid, cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql index f8d96123ce25..aa0d9c1cb20b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql index 77e7bd31c512..d69a8acbadec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_conflict_stream_name_2_hashid, {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias -where conflict_stream_name is not null -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql index 50d9169b110d..dc826e7f53c2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_conflict_stream_name_2_hashid, cast(groups as {{ dbt_utils.type_string() }}) as groups, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql index 2f827f7347b8..9b7dfbfccab7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2') }} tmp -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql index be57970e399e..65a84615e748 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql index cf0d9deb51cd..6205a85370ec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('conflict_stream_scalar_ab1') }} -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql index 781c4bbfc391..42f6c144d3c6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('conflict_stream_scalar_ab2') }} tmp -- conflict_stream_scalar +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql index aaa4eb63f243..58d09347649a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql index 63bd774a716f..9369b7333ece 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -1,10 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab1') }} -- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql deleted file mode 100644 index e2c9a1304fde..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} tmp --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql index fb632de52d62..fc17127ac094 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as data, {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -where {{ adapter.quote('partition') }} is not null -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and {{ adapter.quote('partition') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql index 7ade5b21f875..aa1074726da6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2.sql @@ -1,11 +1,19 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, double_array_data, data, {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql index 0a034922317e..43e9f45bf5f1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +15,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2') }} tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql index 06114ee71d7e..f3478375aeaf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', adapter.quote('column`_\'with""_quotes')) }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} -where {{ adapter.quote('column`_\'with""_quotes') }} is not null -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column`_\'with""_quotes') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql index 076b4bfc975c..ae54378f3109 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql index d4d5028ad31d..0fe19cecbd7c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2') }} tmp -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql index cfacb333734f..791d76b4f4e3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('data'), ['currency'], ['currency']) }} as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'data') }} -where data is not null -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('partition', 'data') }} +where 1 = 1 +and data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql index b4092b9210a8..01ca205eece7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(currency as {{ dbt_utils.type_string() }}) as currency, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1') }} -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql index 69c04aca4634..377ca6ed0320 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2') }} tmp -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql index 48b7ea236ae1..5268dd0dbd4f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('nested_stream_with_complex_columns_resulting_into_long_names_partition', 'partition', 'double_array_data') }} select _airbyte_partition_hashid, {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} as table_alias -{{ cross_join_unnest('partition', 'double_array_data') }} -where double_array_data is not null -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('partition', 'double_array_data') }} +where 1 = 1 +and double_array_data is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql index a3e29e92dfb7..cb8eb9675598 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_partition_hashid, cast(id as {{ dbt_utils.type_string() }}) as id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql index 13295b47f97f..3972432d4691 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2') }} tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql index fd2fbdd54863..a80e0a1c756b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql index e4ed8d075d22..a8aee99469e5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab1') }} -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql index cd80649af632..f55f72315f8a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab2') }} tmp -- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql index 15b34c1ca7d4..d4f394ed7911 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql index a416538b5422..2ccb9349d718 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, children, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_ab1') }} -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql index 0c6703933b10..68c9c6064d65 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,6 @@ select tmp.* from {{ ref('unnest_alias_ab2') }} tmp -- unnest_alias +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql index 57abeb23159f..1a67101d4ea8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} select _airbyte_unnest_alias_hashid, {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias') }} as table_alias -{{ cross_join_unnest('unnest_alias', 'children') }} -where children is not null -- children at unnest_alias/children +{{ cross_join_unnest('unnest_alias', 'children') }} +where 1 = 1 +and children is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql index 8db853c1d601..4c4c2367a5f6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_unnest_alias_hashid, cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, cast(owner as {{ type_json() }}) as owner, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_ab1') }} -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql index 9d84f3be6aa0..06e2a4e3a52e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +14,5 @@ select tmp.* from {{ ref('unnest_alias_children_ab2') }} tmp -- children at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql index 4f28c4f8b5cb..0ed42cb28690 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _airbyte_children_hashid, {{ json_extract_scalar('owner', ['owner_id'], ['owner_id']) }} as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children') }} as table_alias -where owner is not null -- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql index a0f474c62db7..f6b016bf5e78 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _airbyte_children_hashid, cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('unnest_alias_children_owner_ab1') }} -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql index c5f6562c34c2..1a77db63b778 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('unnest_alias_children_owner_ab2') }} tmp -- owner at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql index e3ec7ffd2f82..7ae7824eab31 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql index c437ca698089..3f8824570d2d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_string() }}) as id, cast(date as {{ dbt_utils.type_string() }}) as date, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab1') }} -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql index 727eabab6ac2..0b1c938f2e3e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization_namespace", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab2') }} tmp -- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..80606e29b92c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,108 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab3') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + date, + {{ adapter.quote('partition') }}, + date as _airbyte_start_at, + lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + date, + {{ adapter.quote('partition') }}, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql similarity index 55% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql index 45aafa818f6f..06d33b00087f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -1,12 +1,22 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, date, {{ adapter.quote('partition') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..10f4b807149e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,19 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + id, + children, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index 9ffda6086e35..328153a742fe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_array, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_array_hashid from {{ ref('conflict_stream_array_ab3') }} -- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 8a83ccc01e31..51af86be3918 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_hashid from {{ ref('conflict_stream_name_ab3') }} -- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 94a9dc3d0139..df29869e1b30 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_hashid, conflict_stream_name, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_2_hashid from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql index 1a5bd998fa99..78325474c27a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_conflict_stream_name_2_hashid, groups, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_name_3_hashid from {{ ref('conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3') }} -- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index 9d7acc846fd0..e00b45e8686a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, conflict_stream_scalar, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_conflict_stream_scalar_hashid from {{ ref('conflict_stream_scalar_ab3') }} -- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index 8e228ffad1ab..6e3926b47284 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -1,12 +1,20 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, double_array_data, data, {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_partition_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3') }} -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql index 2bd228318b0d..e8d0f08669f7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_column___with__quotes_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3') }} -- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql index bdcfcf86eb2e..2186017e2c85 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, currency, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_data_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3') }} -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index 50b39ee05b38..7a2ff26eb0f0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_partition_hashid, id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_double_array_data_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3') }} -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql deleted file mode 100644 index 06311da05fc6..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - date, - {{ adapter.quote('partition') }}, - date as _airbyte_start_at, - lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab4') }} --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql index 1004a9e2f4a4..c106d56f953c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid from {{ ref('non_nested_stream_without_namespace_resulting_into_long_names_ab3') }} -- non_nested_stream_without_namespace_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql deleted file mode 100644 index 7a570c3836cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- Final base SQL model -select - id, - children, - _airbyte_emitted_at, - _airbyte_unnest_alias_hashid -from {{ ref('unnest_alias_ab3') }} --- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index 77b2b0ad854e..9c5ef61b6354 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,11 +1,19 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_unnest_alias_hashid, ab_id, owner, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_children_hashid from {{ ref('unnest_alias_children_ab3') }} -- children at unnest_alias/children from {{ ref('unnest_alias') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 0ccf8e33d8d6..322eb9d5f0f6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization", tags=["nested"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "nested" ] +) }} -- Final base SQL model select _airbyte_children_hashid, owner_id, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_owner_hashid from {{ ref('unnest_alias_children_owner_ab3') }} -- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql index ce74645f7864..08f4dde7545a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -1,10 +1,18 @@ -{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization_namespace", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, date, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid from {{ ref('simple_stream_with_namespace_resulting_into_long_names_ab3') }} -- simple_stream_with_namespace_resulting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql index 23ffaa321bd8..aaf10fa8e809 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +14,6 @@ select tmp.* from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_ab2') }} tmp -- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql new file mode 100644 index 000000000000..b3b47010b2a0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql new file mode 100644 index 000000000000..d307b3232ef7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") + ( + select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..3c3524996abe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/unnest_alias.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."unnest_alias" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "unnest_alias__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."unnest_alias" ("id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid") + ( + select "id", "children", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_unnest_alias_hashid" + from "unnest_alias__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..6a49f0ff964e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,54 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_array__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_array_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + json_extract_path_text(_airbyte_data, 'conflict_stream_array', true) as conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +where 1 = 1 +), __dbt__cte__conflict_stream_array_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_array_ab1 +-- conflict_stream_array +where 1 = 1 +), __dbt__cte__conflict_stream_array_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_array as varchar), '') as varchar)) as _airbyte_conflict_stream_array_hashid, + tmp.* +from __dbt__cte__conflict_stream_array_ab2 tmp +-- conflict_stream_array +where 1 = 1 +)-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_array_hashid +from __dbt__cte__conflict_stream_array_ab3 +-- conflict_stream_array from "integrationtests".test_normalization._airbyte_raw_conflict_stream_array +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..13aba9129a52 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,56 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_name__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + + case when json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'conflict_stream_name', true) end + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(conflict_stream_name as varchar) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_ab1 +-- conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_ab2 tmp +-- conflict_stream_name +where 1 = 1 +)-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_hashid +from __dbt__cte__conflict_stream_name_ab3 +-- conflict_stream_name from "integrationtests".test_normalization._airbyte_raw_conflict_stream_name +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..bab7f7d138d5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,57 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + + case when json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) != '' then json_extract_path_text(table_alias.conflict_stream_name, 'conflict_stream_name', true) end + as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null +), __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as varchar) as conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_hashid as varchar), '') || '-' || coalesce(cast(conflict_stream_name as varchar), '') as varchar)) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +where 1 = 1 +)-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_2_hashid +from __dbt__cte__conflict_stream_name_conflict_stream_name_ab3 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..19ac73995aa9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,55 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name_conflict_stream_name__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + case when json_extract_path_text(conflict_stream_name, 'groups', true) != '' then json_extract_path_text(conflict_stream_name, 'groups', true) end as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and conflict_stream_name is not null +), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as varchar) as groups, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +), __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as varchar), '') || '-' || coalesce(cast(groups as varchar), '') as varchar)) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +)-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_name_3_hashid +from __dbt__cte__conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "integrationtests".test_normalization."conflict_stream_name_conflict_stream_name" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..bd6a7edaa276 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,56 @@ + + + create table + "integrationtests".test_normalization."conflict_stream_scalar__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__conflict_stream_scalar_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) != '' then json_extract_path_text(_airbyte_data, 'conflict_stream_scalar', true) end as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +where 1 = 1 +), __dbt__cte__conflict_stream_scalar_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__conflict_stream_scalar_ab1 +-- conflict_stream_scalar +where 1 = 1 +), __dbt__cte__conflict_stream_scalar_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(conflict_stream_scalar as varchar), '') as varchar)) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from __dbt__cte__conflict_stream_scalar_ab2 tmp +-- conflict_stream_scalar +where 1 = 1 +)-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_conflict_stream_scalar_hashid +from __dbt__cte__conflict_stream_scalar_ab3 +-- conflict_stream_scalar from "integrationtests".test_normalization._airbyte_raw_conflict_stream_scalar +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..f691cc5da8fb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,61 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + json_extract_path_text("partition", 'double_array_data', true) as double_array_data, + json_extract_path_text("partition", 'DATA', true) as data, + json_extract_path_text("partition", 'column`_''with"_quotes', true) as "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" as table_alias +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and "partition" is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + data, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid as varchar), '') || '-' || coalesce(cast(double_array_data as varchar), '') || '-' || coalesce(cast(data as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') as varchar)) as _airbyte_partition_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2 tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +)-- Final base SQL model +select + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + double_array_data, + data, + "column`_'with""_quotes", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_partition_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql new file mode 100644 index 000000000000..9eb1e109077e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes.sql @@ -0,0 +1,100 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text("column`_'with""_quotes", numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition."column`_'with""_quotes" + where numbers.generated_number <= json_array_length("column`_'with""_quotes", true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and "column`_'with""_quotes" is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as varchar) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab1 +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_column___with__quotes_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab2 tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_column___with__quotes_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_column___with__quotes_ab3 +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql new file mode 100644 index 000000000000..ed667f0e4087 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -0,0 +1,100 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_data__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 1 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text(data, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.data + where numbers.generated_number <= json_array_length(data, true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'currency', true) != '' then json_extract_path_text(_airbyte_nested_data, 'currency', true) end as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and data is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as varchar) as currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(currency as varchar), '') as varchar)) as _airbyte_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2 tmp +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_data_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..51f7dcf22ca2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,100 @@ + + + create table + "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + +), +joined as ( + select + _airbyte_partition_hashid as _airbyte_hashid, + json_extract_array_element_text(double_array_data, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in nested_stream_with_complex_columns_resulting_into_long_names_partition.double_array_data + where numbers.generated_number <= json_array_length(double_array_data, true) +) +select + _airbyte_partition_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'id', true) end as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +left join joined on _airbyte_partition_hashid = joined._airbyte_hashid +where 1 = 1 +and double_array_data is not null +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as varchar) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_partition_hashid as varchar), '') || '-' || coalesce(cast(id as varchar), '') as varchar)) as _airbyte_double_array_data_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2 tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 +)-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_double_array_data_hashid +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..95f350d3abdd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/non_nested_stream_without_namespace_resulting_into_long_names.sql @@ -0,0 +1,54 @@ + + + create table + "integrationtests".test_normalization."non_nested_stream_without_namespace_resulting_into_long_names__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(date as varchar) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab1 +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid, + tmp.* +from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab2 tmp +-- non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 +)-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_non_nested_stream_without_namespace_resulting_into_long_names_hashid +from __dbt__cte__non_nested_stream_without_namespace_resulting_into_long_names_ab3 +-- non_nested_stream_without_namespace_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..aca1269c8d5f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,107 @@ + + + create table + "integrationtests".test_normalization."unnest_alias_children__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__unnest_alias_children_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +with numbers as ( + + + + + with p as ( + select 0 as generated_number union all select 1 + ), unioned as ( + + select + + + p0.generated_number * power(2, 0) + + + + 1 + as generated_number + + from + + + p as p0 + + + + ) + + select * + from unioned + where generated_number <= 2 + order by generated_number + + +), +joined as ( + select + _airbyte_unnest_alias_hashid as _airbyte_hashid, + json_extract_array_element_text(children, numbers.generated_number::int - 1, true) as _airbyte_nested_data + from "integrationtests".test_normalization."unnest_alias" + cross join numbers + -- only generate the number of records in the cross join that corresponds + -- to the number of items in unnest_alias.children + where numbers.generated_number <= json_array_length(children, true) +) +select + _airbyte_unnest_alias_hashid, + case when json_extract_path_text(_airbyte_nested_data, 'ab_id', true) != '' then json_extract_path_text(_airbyte_nested_data, 'ab_id', true) end as ab_id, + + case when json_extract_path_text(_airbyte_nested_data, 'owner', true) != '' then json_extract_path_text(_airbyte_nested_data, 'owner', true) end + as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."unnest_alias" as table_alias +-- children at unnest_alias/children +left join joined on _airbyte_unnest_alias_hashid = joined._airbyte_hashid +where 1 = 1 +and children is not null +), __dbt__cte__unnest_alias_children_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast(owner as varchar) as owner, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_children_ab1 +-- children at unnest_alias/children +where 1 = 1 +), __dbt__cte__unnest_alias_children_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_unnest_alias_hashid as varchar), '') || '-' || coalesce(cast(ab_id as varchar), '') || '-' || coalesce(cast(owner as varchar), '') as varchar)) as _airbyte_children_hashid, + tmp.* +from __dbt__cte__unnest_alias_children_ab2 tmp +-- children at unnest_alias/children +where 1 = 1 +)-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_children_hashid +from __dbt__cte__unnest_alias_children_ab3 +-- children at unnest_alias/children from "integrationtests".test_normalization."unnest_alias" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..d6c293674dbe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,57 @@ + + + create table + "integrationtests".test_normalization."unnest_alias_children_owner__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__unnest_alias_children_owner_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + case when json_extract_path_text(owner, 'owner_id', true) != '' then json_extract_path_text(owner, 'owner_id', true) end as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization."unnest_alias_children" as table_alias +-- owner at unnest_alias/children/owner +where 1 = 1 +and owner is not null +), __dbt__cte__unnest_alias_children_owner_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__unnest_alias_children_owner_ab1 +-- owner at unnest_alias/children/owner +where 1 = 1 +), __dbt__cte__unnest_alias_children_owner_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_airbyte_children_hashid as varchar), '') || '-' || coalesce(cast(owner_id as varchar), '') as varchar)) as _airbyte_owner_hashid, + tmp.* +from __dbt__cte__unnest_alias_children_owner_ab2 tmp +-- owner at unnest_alias/children/owner +where 1 = 1 +)-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_owner_hashid +from __dbt__cte__unnest_alias_children_owner_ab3 +-- owner at unnest_alias/children/owner from "integrationtests".test_normalization."unnest_alias_children" +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql new file mode 100644 index 000000000000..69dff6931661 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_tables/test_normalization_namespace/simple_stream_with_namespace_resulting_into_long_names.sql @@ -0,0 +1,54 @@ + + + create table + "integrationtests".test_normalization_namespace."simple_stream_with_namespace_resulting_into_long_names__dbt_tmp" + + + compound sortkey(_airbyte_emitted_at) + as ( + +with __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(date as varchar) as date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab1 +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 +), __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') as varchar)) as _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid, + tmp.* +from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab2 tmp +-- simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 +)-- Final base SQL model +select + id, + date, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_simple_stream_with_namespace_resulting_into_long_names_hashid +from __dbt__cte__simple_stream_with_namespace_resulting_into_long_names_ab3 +-- simple_stream_with_namespace_resulting_into_long_names from "integrationtests".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +where 1 = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql new file mode 100644 index 000000000000..361919a65bc2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_views/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab3.sql @@ -0,0 +1,43 @@ + + + create view "integrationtests"._airbyte_test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_ab3__dbt_tmp" as ( + +with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + + case when json_extract_path_text(table_alias._airbyte_data, 'partition', true) != '' then json_extract_path_text(table_alias._airbyte_data, 'partition', true) end + as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +), __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as varchar) as id, + cast(date as varchar) as date, + cast("partition" as varchar) as "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab1 +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast("partition" as varchar), '') as varchar)) as _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, + tmp.* +from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_ab2 tmp +-- nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index e0924e6bbf13..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,84 +0,0 @@ - - - create table - "integrationtests".test_normalization."dedup_cdc_excluded_scd__dbt_tmp" - - - as ( - -with __dbt__cte__dedup_cdc_excluded_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -select - case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, - case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, - case when json_extract_path_text(_airbyte_data, 'column`_''with"_quotes', true) != '' then json_extract_path_text(_airbyte_data, 'column`_''with"_quotes', true) end as "column`_'with""_quotes", - case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, - case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, - _airbyte_emitted_at -from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias --- dedup_cdc_excluded -), __dbt__cte__dedup_cdc_excluded_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -select - cast(id as - bigint -) as id, - cast(name as varchar) as name, - cast("column`_'with""_quotes" as varchar) as "column`_'with""_quotes", - cast(_ab_cdc_lsn as - float -) as _ab_cdc_lsn, - cast(_ab_cdc_updated_at as - float -) as _ab_cdc_updated_at, - cast(_ab_cdc_deleted_at as - float -) as _ab_cdc_deleted_at, - _airbyte_emitted_at -from __dbt__cte__dedup_cdc_excluded_ab1 --- dedup_cdc_excluded -), __dbt__cte__dedup_cdc_excluded_ab3 as ( - --- SQL model to build a hash column based on the values of this record -select - md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast("column`_'with""_quotes" as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from __dbt__cte__dedup_cdc_excluded_ab2 tmp --- dedup_cdc_excluded -), __dbt__cte__dedup_cdc_excluded_ab4 as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from __dbt__cte__dedup_cdc_excluded_ab3 tmp --- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - "column`_'with""_quotes", - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from __dbt__cte__dedup_cdc_excluded_ab4 --- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_row_num = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..354cbd13be02 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,77 @@ + + + + create table + "integrationtests"."test_normalization"."dedup_cdc_excluded_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."dedup_cdc_excluded_ab3" + -- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar), cast(_ab_cdc_updated_at as varchar) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..c9ec8d0622ff --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,83 @@ + + + + create table + "integrationtests"."test_normalization"."dedup_exchange_rate_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_ab3" + -- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') as varchar)) as _airbyte_unique_key, + id, + currency, + date, + timestamp_col, + "hkd@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + date as _airbyte_start_at, + lag(date) over ( + partition by id, currency, cast(nzd as varchar) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id, currency, cast(nzd as varchar) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + date, + timestamp_col, + "hkd@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..c63c52ddd0e7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,69 @@ + + + + create table + "integrationtests"."test_normalization"."renamed_dedup_cdc_excluded_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3" + -- renamed_dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql similarity index 57% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql index 9c685fb4b9b3..688fbdcbe378 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -1,22 +1,29 @@ + create table - "integrationtests".test_normalization."dedup_cdc_excluded__dbt_tmp" + "integrationtests"."test_normalization"."dedup_cdc_excluded__dbt_tmp" + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) as ( -- Final base SQL model select + _airbyte_unique_key, id, name, - "column`_'with""_quotes", _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_dedup_cdc_excluded_hashid from "integrationtests".test_normalization."dedup_cdc_excluded_scd" -- dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded -where _airbyte_active_row = 1 - ); \ No newline at end of file +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 550f24e063f0..21a16d10606d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,13 +1,16 @@ + create table - "integrationtests".test_normalization."dedup_exchange_rate__dbt_tmp" + "integrationtests"."test_normalization"."dedup_exchange_rate__dbt_tmp" + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) as ( -- Final base SQL model select + _airbyte_unique_key, id, currency, date, @@ -16,9 +19,14 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from "integrationtests".test_normalization."dedup_exchange_rate_scd" -- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_active_row = 1 - ); \ No newline at end of file +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql index e0cab59bb19e..a560a42d546a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -1,9 +1,11 @@ + create table - "integrationtests".test_normalization."exchange_rate__dbt_tmp" + "integrationtests"."test_normalization"."exchange_rate__dbt_tmp" + compound sortkey(_airbyte_emitted_at) as ( with __dbt__cte__exchange_rate_ab1 as ( @@ -18,9 +20,13 @@ select case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization._airbyte_raw_exchange_rate as table_alias -- exchange_rate +where 1 = 1 + ), __dbt__cte__exchange_rate_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -45,9 +51,13 @@ select cast(usd as float ) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__exchange_rate_ab1 -- exchange_rate +where 1 = 1 + ), __dbt__cte__exchange_rate_ab3 as ( -- SQL model to build a hash column based on the values of this record @@ -56,6 +66,8 @@ select tmp.* from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate +where 1 = 1 + )-- Final base SQL model select id, @@ -66,8 +78,13 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from __dbt__cte__exchange_rate_ab3 -- exchange_rate from "integrationtests".test_normalization._airbyte_raw_exchange_rate - ); \ No newline at end of file +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..035acaa17f2a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,25 @@ + + + + create table + "integrationtests"."test_normalization"."renamed_dedup_cdc_excluded__dbt_tmp" + + + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" +-- renamed_dedup_cdc_excluded from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..a9657af6c9d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,77 @@ + + + create table + "integrationtests".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar), cast(_ab_cdc_updated_at as varchar), cast(_ab_cdc_log_pos as varchar) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql similarity index 70% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index 1ed954b4633c..6bc840ab2ff4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -4,19 +4,24 @@ "integrationtests".test_normalization."pos_dedup_cdcx__dbt_tmp" + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) as ( -- Final base SQL model select + _airbyte_unique_key, id, name, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from "integrationtests".test_normalization."pos_dedup_cdcx_scd" -- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..aef3d5e083d6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,53 @@ + + + create view "integrationtests"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( + +with __dbt__cte__dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as varchar) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__dedup_cdc_excluded_ab1 +-- dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__dedup_cdc_excluded_ab2 tmp +-- dedup_cdc_excluded +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 64% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 96fdd13d575e..f543d9f4bbb6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,10 +1,6 @@ - create table - "integrationtests".test_normalization."dedup_exchange_rate_scd__dbt_tmp" - - - as ( + create view "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( with __dbt__cte__dedup_exchange_rate_ab1 as ( @@ -18,9 +14,13 @@ select case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate +where 1 = 1 + ), __dbt__cte__dedup_exchange_rate_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -45,50 +45,19 @@ select cast(usd as float ) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__dedup_exchange_rate_ab1 -- dedup_exchange_rate -), __dbt__cte__dedup_exchange_rate_ab3 as ( +where 1 = 1 --- SQL model to build a hash column based on the values of this record +)-- SQL model to build a hash column based on the values of this record select md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(hkd_special___characters as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') as varchar)) as _airbyte_dedup_exchange_rate_hashid, tmp.* from __dbt__cte__dedup_exchange_rate_ab2 tmp -- dedup_exchange_rate -), __dbt__cte__dedup_exchange_rate_ab4 as ( +where 1 = 1 --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from __dbt__cte__dedup_exchange_rate_ab3 tmp --- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - date, - timestamp_col, - "hkd@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - date as _airbyte_start_at, - lag(date) over ( - partition by id, currency, cast(nzd as varchar) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id, currency, cast(nzd as varchar) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from __dbt__cte__dedup_exchange_rate_ab4 --- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate -where _airbyte_row_num = 1 - ); \ No newline at end of file + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 58% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index 3871ad68a7f4..3502913a583f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/final/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -1,10 +1,6 @@ - create table - "integrationtests".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" - - - as ( + create view "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( with __dbt__cte__pos_dedup_cdcx_ab1 as ( @@ -16,9 +12,12 @@ select case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, case when json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) end as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias -- pos_dedup_cdcx +where 1 = 1 ), __dbt__cte__pos_dedup_cdcx_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type @@ -39,48 +38,17 @@ select cast(_ab_cdc_log_pos as float ) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at from __dbt__cte__pos_dedup_cdcx_ab1 -- pos_dedup_cdcx -), __dbt__cte__pos_dedup_cdcx_ab3 as ( - --- SQL model to build a hash column based on the values of this record +where 1 = 1 +)-- SQL model to build a hash column based on the values of this record select md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_pos_dedup_cdcx_hashid, tmp.* from __dbt__cte__pos_dedup_cdcx_ab2 tmp -- pos_dedup_cdcx -), __dbt__cte__pos_dedup_cdcx_ab4 as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from __dbt__cte__pos_dedup_cdcx_ab3 tmp --- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx -)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from __dbt__cte__pos_dedup_cdcx_ab4 --- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx -where _airbyte_row_num = 1 - ); \ No newline at end of file +where 1 = 1 + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..e7841d06e0fe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,39 @@ + + + create view "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__renamed_dedup_cdc_excluded_ab1 +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index 6144a13617e8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('exchange_rate') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('dedup_exchange_rate_scd') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('dedup_exchange_rate') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('dedup_cdc_excluded_scd') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('dedup_cdc_excluded') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('pos_dedup_cdcx_scd') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('pos_dedup_cdcx') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql index 71f6448f6532..98cd148ff512 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql index 2f1c85715033..b808389ede5a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -1,13 +1,21 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, cast(name as {{ dbt_utils.type_string() }}) as name, - cast({{ adapter.quote('column`_\'with""_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column`_\'with""_quotes') }}, cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_cdc_excluded_ab1') }} -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql deleted file mode 100644 index 4e6d97a65757..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to build a hash column based on the values of this record -select - {{ dbt_utils.surrogate_key([ - 'id', - 'name', - adapter.quote('column`_\'with""_quotes'), - '_ab_cdc_lsn', - '_ab_cdc_updated_at', - '_ab_cdc_deleted_at', - ]) }} as _airbyte_dedup_cdc_excluded_hashid, - tmp.* -from {{ ref('dedup_cdc_excluded_ab2') }} tmp --- dedup_cdc_excluded - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql deleted file mode 100644 index aa34179c3c5c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_cdc_excluded_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_cdc_excluded_ab3') }} tmp --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 5dcea3571341..463aebe85f6b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 9e15010eb3b5..35914dac7dcc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +14,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('dedup_exchange_rate_ab1') }} -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql deleted file mode 100644 index ef5f6e52d60e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_dedup_exchange_rate_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('dedup_exchange_rate_ab3') }} tmp --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 70419151a6dc..354a14427e50 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -9,7 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index 907c04af5eb4..8096d935aeaa 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -9,7 +14,11 @@ select cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, cast(nzd as {{ dbt_utils.type_float() }}) as nzd, cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('exchange_rate_ab1') }} -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index d1b84842e82f..a07ee57a28a3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +19,6 @@ select tmp.* from {{ ref('exchange_rate_ab2') }} tmp -- exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql index 1041493c6c67..9d9592bf5bcc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, @@ -7,7 +12,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} as table_alias -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql index bccb189a9d7a..32dea2cfa7c8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(id as {{ dbt_utils.type_bigint() }}) as id, @@ -7,7 +12,10 @@ select cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, cast(_ab_cdc_log_pos as {{ dbt_utils.type_float() }}) as _ab_cdc_log_pos, - _airbyte_emitted_at + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at from {{ ref('pos_dedup_cdcx_ab1') }} -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql deleted file mode 100644 index 7c897124b395..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _airbyte_pos_dedup_cdcx_hashid - order by _airbyte_emitted_at asc - ) as _airbyte_row_num, - tmp.* -from {{ ref('pos_dedup_cdcx_ab3') }} tmp --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..c324b5203905 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab1.sql @@ -0,0 +1,17 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..df5c1035dd92 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -0,0 +1,17 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('renamed_dedup_cdc_excluded_ab1') }} +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..19d4d181c449 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,112 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_cdc_excluded_ab3') }} + -- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..12f992cd0bbd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,122 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('dedup_exchange_rate_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('dedup_exchange_rate_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('dedup_exchange_rate_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('dedup_exchange_rate_ab3') }} + -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + 'nzd', + ]) }} as _airbyte_unique_key, + id, + currency, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + date as _airbyte_start_at, + lag(date) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(date) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by + date is null asc, + date desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + currency, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_exchange_rate_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..500c257b4a70 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,104 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('renamed_dedup_cdc_excluded_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('renamed_dedup_cdc_excluded_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('renamed_dedup_cdc_excluded_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('renamed_dedup_cdc_excluded_ab3') }} + -- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_renamed_dedup_cdc_excluded_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..385a2a1c6b24 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,24 @@ +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_dedup_cdc_excluded_hashid +from {{ ref('dedup_cdc_excluded_scd') }} +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 51% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 0e3b855088c3..64467e8148ab 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,6 +1,12 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, currency, date, @@ -9,9 +15,13 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from {{ ref('dedup_exchange_rate_scd') }} -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql similarity index 53% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql index ffe5cafe5e4b..62dbd07d0962 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/exchange_rate.sql @@ -1,4 +1,9 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select id, @@ -9,8 +14,12 @@ select hkd_special___characters, nzd, usd, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from {{ ref('exchange_rate_ab3') }} -- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..12a26f6e4326 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,20 @@ +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _airbyte_unique_key, + id, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_renamed_dedup_cdc_excluded_hashid +from {{ ref('renamed_dedup_cdc_excluded_scd') }} +-- renamed_dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_renamed_dedup_cdc_excluded') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..5140caa6d9f4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,114 @@ +{{ config( + sort = ["_airbyte_active_row", "_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at') }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('pos_dedup_cdcx_ab3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('pos_dedup_cdcx_ab3') }} as inc_data on 1 = 0 + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from new_data + union all + select {{ dbt_utils.star(ref('pos_dedup_cdcx_ab3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('pos_dedup_cdcx_ab3') }} + -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_updated_at as {{ dbt_utils.type_string() }}), cast(_ab_cdc_log_pos as {{ dbt_utils.type_string() }}) + order by _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at', '_ab_cdc_deleted_at', '_ab_cdc_updated_at', '_ab_cdc_log_pos' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql deleted file mode 100644 index 71c78bd0cb26..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - {{ adapter.quote('column`_\'with""_quotes') }}, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_cdc_excluded_hashid -from {{ ref('dedup_cdc_excluded_ab4') }} --- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 63fab0795ade..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - currency, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - date as _airbyte_start_at, - lag(date) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(date) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by date is null asc, date desc, _airbyte_emitted_at desc - ) is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_ab4') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql index ed02a5d04940..1786654bece8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -1,15 +1,24 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} +{{ config( + sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _airbyte_unique_key, id, name, _ab_cdc_lsn, _ab_cdc_updated_at, _ab_cdc_deleted_at, _ab_cdc_log_pos, + _airbyte_ab_id, _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_pos_dedup_cdcx_hashid from {{ ref('pos_dedup_cdcx_scd') }} -- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql deleted file mode 100644 index 81216ec938ee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/pos_dedup_cdcx_scd.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="test_normalization", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - id, - name, - _ab_cdc_lsn, - _ab_cdc_updated_at, - _ab_cdc_deleted_at, - _ab_cdc_log_pos, - _airbyte_emitted_at as _airbyte_start_at, - lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when lag(_airbyte_emitted_at) over ( - partition by id - order by _airbyte_emitted_at is null asc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc - ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, - _airbyte_emitted_at, - _airbyte_pos_dedup_cdcx_hashid -from {{ ref('pos_dedup_cdcx_ab4') }} --- pos_dedup_cdcx from {{ source('test_normalization', '_airbyte_raw_pos_dedup_cdcx') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql similarity index 56% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql index 6fa998e293a0..1e63c2ad6c92 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -1,10 +1,14 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ 'id', 'name', - 'column___with__quotes', '_ab_cdc_lsn', '_ab_cdc_updated_at', '_ab_cdc_deleted_at', @@ -12,4 +16,6 @@ select tmp.* from {{ ref('dedup_cdc_excluded_ab2') }} tmp -- dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql index 599a1ccec515..e92210d17f06 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +19,6 @@ select tmp.* from {{ ref('dedup_exchange_rate_ab2') }} tmp -- dedup_exchange_rate +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql similarity index 62% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql index 0fd3196cea9f..e82906073868 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_ctes/test_normalization/pos_dedup_cdcx_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -12,4 +17,5 @@ select tmp.* from {{ ref('pos_dedup_cdcx_ab2') }} tmp -- pos_dedup_cdcx +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..b59cdd1d2b2c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,17 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_airbyte_ab_id'), + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + ]) }} as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('renamed_dedup_cdc_excluded_ab2') }} tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml index c085ca8ccc25..45c338b893ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _airbyte_raw_dedup_exchange_rate - name: _airbyte_raw_exchange_rate - name: _airbyte_raw_pos_dedup_cdcx + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..b9485899f9c3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..53aa2cf9aca6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_exchange_rate_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "dedup_exchange_rate_scd__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from "dedup_exchange_rate_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..70ebdfca1c57 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..8f1ca62ed990 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_cdc_excluded__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_cdc_excluded" ("_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid" + from "dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..977b187c317b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_exchange_rate" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_exchange_rate__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + ( + select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from "dedup_exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..0da80f1f8245 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."exchange_rate" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "exchange_rate__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."exchange_rate" ("id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") + ( + select "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid" + from "exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..94ffdc307498 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."renamed_dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + ( + select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql new file mode 100644 index 000000000000..a9657af6c9d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -0,0 +1,77 @@ + + + create table + "integrationtests".test_normalization."pos_dedup_cdcx_scd__dbt_tmp" + + + compound sortkey(_airbyte_active_row,_airbyte_unique_key,_airbyte_emitted_at) + as ( + +with + +input_data as ( + select * + from "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3" + -- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc, _ab_cdc_updated_at desc, _ab_cdc_log_pos desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_pos_dedup_cdcx_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _airbyte_unique_key, _airbyte_start_at, _airbyte_emitted_at, cast(_ab_cdc_deleted_at as varchar), cast(_ab_cdc_updated_at as varchar), cast(_ab_cdc_log_pos as varchar) + order by _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as varchar), '') || '-' || coalesce(cast(_airbyte_start_at as varchar), '') || '-' || coalesce(cast(_airbyte_emitted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from dedup_data where _airbyte_row_num = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql new file mode 100644 index 000000000000..6bc840ab2ff4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/pos_dedup_cdcx.sql @@ -0,0 +1,27 @@ + + + create table + "integrationtests".test_normalization."pos_dedup_cdcx__dbt_tmp" + + + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) + as ( + +-- Final base SQL model +select + _airbyte_unique_key, + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_pos_dedup_cdcx_hashid +from "integrationtests".test_normalization."pos_dedup_cdcx_scd" +-- pos_dedup_cdcx from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx +where 1 = 1 +and _airbyte_active_row = 1 + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..aef3d5e083d6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,53 @@ + + + create view "integrationtests"._airbyte_test_normalization."dedup_cdc_excluded_ab3__dbt_tmp" as ( + +with __dbt__cte__dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as varchar) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__dedup_cdc_excluded_ab1 +-- dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__dedup_cdc_excluded_ab2 tmp +-- dedup_cdc_excluded +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..f543d9f4bbb6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,63 @@ + + + create view "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( + +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, + case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", + case when json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD_special___characters', true) end as hkd_special___characters, + case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, + case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as varchar) as currency, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("hkd@spéçiäl & characters" as + float +) as "hkd@spéçiäl & characters", + cast(hkd_special___characters as varchar) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(hkd_special___characters as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') as varchar)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from __dbt__cte__dedup_exchange_rate_ab2 tmp +-- dedup_exchange_rate +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql new file mode 100644 index 000000000000..3502913a583f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/pos_dedup_cdcx_ab3.sql @@ -0,0 +1,54 @@ + + + create view "integrationtests"._airbyte_test_normalization."pos_dedup_cdcx_ab3__dbt_tmp" as ( + +with __dbt__cte__pos_dedup_cdcx_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_log_pos', true) end as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_pos_dedup_cdcx as table_alias +-- pos_dedup_cdcx +where 1 = 1 +), __dbt__cte__pos_dedup_cdcx_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as varchar) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + cast(_ab_cdc_log_pos as + float +) as _ab_cdc_log_pos, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__pos_dedup_cdcx_ab1 +-- pos_dedup_cdcx +where 1 = 1 +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_log_pos as varchar), '') as varchar)) as _airbyte_pos_dedup_cdcx_hashid, + tmp.* +from __dbt__cte__pos_dedup_cdcx_ab2 tmp +-- pos_dedup_cdcx +where 1 = 1 + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..e7841d06e0fe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,39 @@ + + + create view "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__renamed_dedup_cdc_excluded_ab1 +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') as varchar)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..35192e5d2a73 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_exchange_rate_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "dedup_exchange_rate_scd__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id" + from "dedup_exchange_rate_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..3fb70e64ecc7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + from "renamed_dedup_cdc_excluded_scd__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..47aaec909812 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."dedup_exchange_rate" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "dedup_exchange_rate__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id") + ( + select "_airbyte_unique_key", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid", "new_column", "id" + from "dedup_exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..3b8d494b22b5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/exchange_rate.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."exchange_rate" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "exchange_rate__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."exchange_rate" ("currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id") + ( + select "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid", "new_column", "id" + from "exchange_rate__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql new file mode 100644 index 000000000000..8d6374d4880e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ + + delete + from "integrationtests".test_normalization."renamed_dedup_cdc_excluded" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + + insert into "integrationtests".test_normalization."renamed_dedup_cdc_excluded" ("_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at") + ( + select "_airbyte_unique_key", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at" + from "renamed_dedup_cdc_excluded__dbt_tmp" + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..765e200b0da1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,65 @@ + + + create view "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_ab3__dbt_tmp" as ( + +with __dbt__cte__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'currency', true) != '' then json_extract_path_text(_airbyte_data, 'currency', true) end as currency, + case when json_extract_path_text(_airbyte_data, 'new_column', true) != '' then json_extract_path_text(_airbyte_data, 'new_column', true) end as new_column, + case when json_extract_path_text(_airbyte_data, 'date', true) != '' then json_extract_path_text(_airbyte_data, 'date', true) end as date, + case when json_extract_path_text(_airbyte_data, 'timestamp_col', true) != '' then json_extract_path_text(_airbyte_data, 'timestamp_col', true) end as timestamp_col, + case when json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) != '' then json_extract_path_text(_airbyte_data, 'HKD@spéçiäl & characters', true) end as "hkd@spéçiäl & characters", + case when json_extract_path_text(_airbyte_data, 'NZD', true) != '' then json_extract_path_text(_airbyte_data, 'NZD', true) end as nzd, + case when json_extract_path_text(_airbyte_data, 'USD', true) != '' then json_extract_path_text(_airbyte_data, 'USD', true) end as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +where 1 = 1 + +), __dbt__cte__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + float +) as id, + cast(currency as varchar) as currency, + cast(new_column as + float +) as new_column, + cast(nullif(date, '') as + date +) as date, + cast(nullif(timestamp_col, '') as + timestamp with time zone +) as timestamp_col, + cast("hkd@spéçiäl & characters" as + float +) as "hkd@spéçiäl & characters", + cast(nzd as + float +) as nzd, + cast(usd as + bigint +) as usd, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(currency as varchar), '') || '-' || coalesce(cast(new_column as varchar), '') || '-' || coalesce(cast(date as varchar), '') || '-' || coalesce(cast(timestamp_col as varchar), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as varchar), '') || '-' || coalesce(cast(nzd as varchar), '') || '-' || coalesce(cast(usd as varchar), '') as varchar)) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from __dbt__cte__dedup_exchange_rate_ab2 tmp +-- dedup_exchange_rate +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..0c4e15f9ecee --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/renamed_dedup_cdc_excluded_ab3.sql @@ -0,0 +1,53 @@ + + + create view "integrationtests"._airbyte_test_normalization."renamed_dedup_cdc_excluded_ab3__dbt_tmp" as ( + +with __dbt__cte__renamed_dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + case when json_extract_path_text(_airbyte_data, 'id', true) != '' then json_extract_path_text(_airbyte_data, 'id', true) end as id, + case when json_extract_path_text(_airbyte_data, 'name', true) != '' then json_extract_path_text(_airbyte_data, 'name', true) end as name, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_lsn', true) end as _ab_cdc_lsn, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_updated_at', true) end as _ab_cdc_updated_at, + case when json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) != '' then json_extract_path_text(_airbyte_data, '_ab_cdc_deleted_at', true) end as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from "integrationtests".test_normalization._airbyte_raw_renamed_dedup_cdc_excluded as table_alias +-- renamed_dedup_cdc_excluded +where 1 = 1 + +), __dbt__cte__renamed_dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as varchar) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at +from __dbt__cte__renamed_dedup_cdc_excluded_ab1 +-- renamed_dedup_cdc_excluded +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(id as varchar), '') || '-' || coalesce(cast(name as varchar), '') || '-' || coalesce(cast(_ab_cdc_lsn as varchar), '') || '-' || coalesce(cast(_ab_cdc_updated_at as varchar), '') || '-' || coalesce(cast(_ab_cdc_deleted_at as varchar), '') as varchar)) as _airbyte_renamed_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__cte__renamed_dedup_cdc_excluded_ab2 tmp +-- renamed_dedup_cdc_excluded +where 1 = 1 + + ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql deleted file mode 100644 index cd4e337de08a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3" tmp --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql deleted file mode 100644 index e1690d039b07..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ /dev/null @@ -1,25 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as - ( --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - DATE, - PARTITION, - DATE as _AIRBYTE_START_AT, - lag(DATE) over ( - partition by ID - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( - partition by ID - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4" --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql index 6bef3219266c..e18e6a208a2c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql @@ -5,7 +5,10 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, get_path(parse_json(_airbyte_data), '"conflict_stream_array"') as CONFLICT_STREAM_ARRAY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY as table_alias -- CONFLICT_STREAM_ARRAY +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql similarity index 74% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql index dada3b7f692c..97e976fd6462 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql @@ -7,7 +7,10 @@ select varchar ) as ID, CONFLICT_STREAM_ARRAY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" -- CONFLICT_STREAM_ARRAY +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql index 729c495f08c2..66a710356b32 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" tmp -- CONFLICT_STREAM_ARRAY +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql index 6759cf235073..1a41bd6a787d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql @@ -7,7 +7,10 @@ select get_path(parse_json(table_alias._airbyte_data), '"conflict_stream_name"') as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME as table_alias -- CONFLICT_STREAM_NAME +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql index 9de606829965..bf735e634dcb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql @@ -9,7 +9,10 @@ select cast(CONFLICT_STREAM_NAME as variant ) as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" -- CONFLICT_STREAM_NAME +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql index a8f985bc6467..3fe0514dbf9f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" tmp -- CONFLICT_STREAM_NAME +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql similarity index 77% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql index 7490c42bcba6..e3ef93a18792 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -7,8 +7,11 @@ select get_path(parse_json(table_alias.CONFLICT_STREAM_NAME), '"conflict_stream_name"') as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as table_alias -where CONFLICT_STREAM_NAME is not null -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql index 2e8959fe5a9f..e9b6b9804fd9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -7,7 +7,10 @@ select cast(CONFLICT_STREAM_NAME as variant ) as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql index 743006f52639..0616d66795ea 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql index d3cc576b8e69..3e793970e09c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -5,8 +5,11 @@ select _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, to_varchar(get_path(parse_json(CONFLICT_STREAM_NAME), '"groups"')) as GROUPS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as table_alias -where CONFLICT_STREAM_NAME is not null -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql index 6518b2b8d815..a405f7059b95 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -7,7 +7,10 @@ select cast(GROUPS as varchar ) as GROUPS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql index cbdd87f1fe23..26ab7e066acf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql similarity index 80% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql index 73122f365f21..d61b09ff806a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql @@ -5,7 +5,10 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, to_varchar(get_path(parse_json(_airbyte_data), '"conflict_stream_scalar"')) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR as table_alias -- CONFLICT_STREAM_SCALAR +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql index 53dafe61398f..703883993977 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql @@ -9,7 +9,10 @@ select cast(CONFLICT_STREAM_SCALAR as bigint ) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" -- CONFLICT_STREAM_SCALAR +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql index e453bd385997..00e7c7886c89 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" tmp -- CONFLICT_STREAM_SCALAR +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql similarity index 84% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql index 320bca5e222a..6e7e79743d05 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -8,7 +8,11 @@ select get_path(parse_json(table_alias._airbyte_data), '"partition"') as PARTITION, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES as table_alias -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql index e6a15b835c01..eb9c920cc525 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -12,7 +12,11 @@ select cast(PARTITION as variant ) as PARTITION, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql index d0d5c60bb47c..7b11ebe9ba64 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql @@ -7,8 +7,11 @@ select get_path(parse_json(PARTITION), '"double_array_data"') as DOUBLE_ARRAY_DATA, get_path(parse_json(PARTITION), '"DATA"') as DATA, get_path(parse_json(PARTITION), '"column`_''with""_quotes"') as "column`_'with""_quotes", - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as table_alias -where PARTITION is not null -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and PARTITION is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql index b2af08cb9c9f..8c9edac47baf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql @@ -7,7 +7,10 @@ select DOUBLE_ARRAY_DATA, DATA, "column`_'with""_quotes", - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql index 1f3f31e025f5..d659cf82cab8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql @@ -17,4 +17,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" tmp -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql index 73e33d8747e7..17a36065ce62 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql @@ -6,9 +6,12 @@ select _AIRBYTE_PARTITION_HASHID, to_varchar(get_path(parse_json("column`_'with""_quotes".value), '"currency"')) as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias -cross join table(flatten("column`_'with""_quotes")) as "column`_'with""_quotes" -where "column`_'with""_quotes" is not null -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +cross join table(flatten("column`_'with""_quotes")) as "column`_'with""_quotes" +where 1 = 1 +and "column`_'with""_quotes" is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql index 37164e52f9c4..de013bb4fb92 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql @@ -7,7 +7,10 @@ select cast(CURRENCY as varchar ) as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql index 9488057417b0..1996258464ce 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" tmp -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql index a1eca419d8e1..fbd05545c768 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql @@ -6,9 +6,12 @@ select _AIRBYTE_PARTITION_HASHID, to_varchar(get_path(parse_json(DATA.value), '"currency"')) as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias -cross join table(flatten(DATA)) as DATA -where DATA is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join table(flatten(DATA)) as DATA +where 1 = 1 +and DATA is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql index d2be8a3d89a8..bdf9cd2b386c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql @@ -7,7 +7,10 @@ select cast(CURRENCY as varchar ) as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql index 0635b936fcf9..c4a6deea96b7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql index 0fcaa6ecd82f..3013309f1f90 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql @@ -6,9 +6,12 @@ select _AIRBYTE_PARTITION_HASHID, to_varchar(get_path(parse_json(DOUBLE_ARRAY_DATA.value), '"id"')) as ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias -cross join table(flatten(DOUBLE_ARRAY_DATA)) as DOUBLE_ARRAY_DATA -where DOUBLE_ARRAY_DATA is not null -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join table(flatten(DOUBLE_ARRAY_DATA)) as DOUBLE_ARRAY_DATA +where 1 = 1 +and DOUBLE_ARRAY_DATA is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql index 8f10bbffe486..5c1e8b226a6a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql @@ -7,7 +7,10 @@ select cast(ID as varchar ) as ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql index 74f6b6e45993..662055a5ee44 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" tmp -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql index d93e36223827..44ffce39889f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -5,7 +5,10 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql index 168beb12710e..437075adbab4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -9,7 +9,10 @@ select cast(DATE as varchar ) as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql index 9a93e64a0b04..70f3a3546307 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql index f58be28c589f..3c6d0a2e24d2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql @@ -5,7 +5,11 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, get_path(parse_json(_airbyte_data), '"children"') as CHILDREN, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS as table_alias -- UNNEST_ALIAS +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql similarity index 72% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql index 5e0ec3de9d76..520bc1d1d7ec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql @@ -7,7 +7,11 @@ select bigint ) as ID, CHILDREN, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" -- UNNEST_ALIAS +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql index bfce4d9064ee..753d624ac210 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql @@ -13,4 +13,6 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" tmp -- UNNEST_ALIAS +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql index a6aa86bdf43b..a4717707483e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql @@ -9,9 +9,12 @@ select get_path(parse_json(CHILDREN.value), '"owner"') as OWNER, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as table_alias -cross join table(flatten(CHILDREN)) as CHILDREN -where CHILDREN is not null -- CHILDREN at unnest_alias/children +cross join table(flatten(CHILDREN)) as CHILDREN +where 1 = 1 +and CHILDREN is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql similarity index 77% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql index 65d17c2515a5..b0fd97c240a0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql @@ -10,7 +10,10 @@ select cast(OWNER as variant ) as OWNER, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" -- CHILDREN at unnest_alias/children +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql index 9eaa2bdd13e8..88b1593cc4d0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql @@ -15,4 +15,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" tmp -- CHILDREN at unnest_alias/children +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql similarity index 75% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql index 6e7d382b882c..806d713ba2f3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql @@ -5,8 +5,11 @@ select _AIRBYTE_CHILDREN_HASHID, to_varchar(get_path(parse_json(OWNER), '"owner_id"')) as OWNER_ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as table_alias -where OWNER is not null -- OWNER at unnest_alias/children/owner +where 1 = 1 +and OWNER is not null ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql similarity index 76% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql index 56d5a85204cd..4e3f70fc2e82 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql @@ -7,7 +7,10 @@ select cast(OWNER_ID as bigint ) as OWNER_ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" -- OWNER at unnest_alias/children/owner +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql similarity index 97% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql index a1a831a0f496..79189e35ffbe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" tmp -- OWNER at unnest_alias/children/owner +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql index 863bca6116b1..dc9d26c10728 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -5,7 +5,10 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql similarity index 79% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql index b9cc58f6026f..ac66f7e51acd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -9,7 +9,10 @@ select cast(DATE as varchar ) as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql index 33af774280d4..0becb20f6e1b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -13,4 +13,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql similarity index 58% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql index 557ae8edf658..54641b78f236 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql @@ -1,16 +1,23 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as - ( + (select * from( + -- Final base SQL model select + _AIRBYTE_UNIQUE_KEY, ID, DATE, PARTITION, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES -where _airbyte_active_row = 1 +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql similarity index 56% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql index 6540b0016cce..49df6fc5d79b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql @@ -1,14 +1,20 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as - ( + (select * from( + -- Final base SQL model select ID, CHILDREN, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_UNNEST_ALIAS_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB3" -- UNNEST_ALIAS from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS +where 1 = 1 + + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql new file mode 100644 index 000000000000..a3f67db69287 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -0,0 +1,81 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3" + -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + DATE, + PARTITION, + DATE as _AIRBYTE_START_AT, + lag(DATE) over ( + partition by ID + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(DATE) over ( + partition by ID + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + DATE, + PARTITION, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql similarity index 59% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql index 21233bc9ddb8..a366f2bba8cc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" as - ( + (select * from( + -- Final base SQL model select ID, CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" -- CONFLICT_STREAM_ARRAY from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql similarity index 58% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql index 5abb9b5bebaf..a20080a7f7f4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as - ( + (select * from( + -- Final base SQL model select ID, CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_NAME_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" -- CONFLICT_STREAM_NAME from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql similarity index 62% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql index 6e44eb56fa7c..4d69fa730172 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as - ( + (select * from( + -- Final base SQL model select _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql similarity index 64% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql index 6c264d164dbe..97d25955385f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as - ( + (select * from( + -- Final base SQL model select _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, GROUPS, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql similarity index 59% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql index f6978d8b376f..efc7cbe9dce6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" as - ( + (select * from( + -- Final base SQL model select ID, CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" -- CONFLICT_STREAM_SCALAR from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql similarity index 67% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql index b3de9a34e619..75228a851232 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql @@ -1,16 +1,21 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as - ( + (select * from( + -- Final base SQL model select _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, DOUBLE_ARRAY_DATA, DATA, "column`_'with""_quotes", + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_PARTITION_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql similarity index 66% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql index c64b65682078..1f2adeefe899 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" as - ( + (select * from( + -- Final base SQL model select _AIRBYTE_PARTITION_HASHID, CURRENCY, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_COLUMN___WITH__QUOTES_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql similarity index 64% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql index b88de2106399..cfc1ce750875 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as - ( + (select * from( + -- Final base SQL model select _AIRBYTE_PARTITION_HASHID, CURRENCY, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_DATA_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql similarity index 65% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql index 2d0123ef9cd4..4bdef077e4b3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as - ( + (select * from( + -- Final base SQL model select _AIRBYTE_PARTITION_HASHID, ID, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql similarity index 64% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql index 28541bcc6f05..a470f14eceac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" as - ( + (select * from( + -- Final base SQL model select ID, DATE, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql similarity index 59% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql index bf03f70901c6..0657b14de313 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql @@ -1,15 +1,20 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as - ( + (select * from( + -- Final base SQL model select _AIRBYTE_UNNEST_ALIAS_HASHID, AB_ID, OWNER, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CHILDREN_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" -- CHILDREN at unnest_alias/children from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql similarity index 59% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql index 7edb0f77df92..652a9bd48931 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" as - ( + (select * from( + -- Final base SQL model select _AIRBYTE_CHILDREN_HASHID, OWNER_ID, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_OWNER_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" -- OWNER at unnest_alias/children/owner from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql similarity index 64% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql index f913181bdf3f..057fea2c9ee7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -1,14 +1,19 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" as - ( + (select * from( + -- Final base SQL model select ID, DATE, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql index 49fb90573503..b224c3ada40e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/final/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/first_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -15,4 +15,6 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" tmp -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index fceb5499a1f2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,19 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 2 as expected_count - from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} -union all - select distinct count(*) as row_count, 2 as expected_count - from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} -union all - select count(distinct currency) as row_count, 1 as expected_count - from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA') }} --- union all --- select count(distinct id) as row_count, 3 as expected_count --- from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql index 61230105c911..991a40b91868 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as CONFLICT_STREAM_ARRAY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_ARRAY') }} as table_alias -- CONFLICT_STREAM_ARRAY +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql index 8978f4dbac41..645bc6ce95f4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, CONFLICT_STREAM_ARRAY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_ARRAY_AB1') }} -- CONFLICT_STREAM_ARRAY +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql index 9c9ee59dc86d..83752bf1352d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('CONFLICT_STREAM_ARRAY_AB2') }} tmp -- CONFLICT_STREAM_ARRAY +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql index 338b614c5cf9..5822167058d6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_NAME') }} as table_alias -- CONFLICT_STREAM_NAME +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql index 9279f897b33d..73e500cda859 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, cast(CONFLICT_STREAM_NAME as {{ type_json() }}) as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_NAME_AB1') }} -- CONFLICT_STREAM_NAME +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql index 7ae3ea76192d..516db74e7a46 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('CONFLICT_STREAM_NAME_AB2') }} tmp -- CONFLICT_STREAM_NAME +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql index 10e877a6018e..8c5659840ae3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, {{ json_extract('table_alias', 'CONFLICT_STREAM_NAME', ['conflict_stream_name'], ['conflict_stream_name']) }} as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_NAME') }} as table_alias -where CONFLICT_STREAM_NAME is not null -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql index 32736a9e7dab..e23598a11d34 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, cast(CONFLICT_STREAM_NAME as {{ type_json() }}) as CONFLICT_STREAM_NAME, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1') }} -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql index fa8fd857e30d..97ca26979ab2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2') }} tmp -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql index 094d1bf989fc..df7f67fc3449 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, {{ json_extract_scalar('CONFLICT_STREAM_NAME', ['groups'], ['groups']) }} as GROUPS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME') }} as table_alias -where CONFLICT_STREAM_NAME is not null -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql index 24867ad66cb7..1956ff59d185 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, cast(GROUPS as {{ dbt_utils.type_string() }}) as GROUPS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1') }} -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql index 9f2fd41df405..e4a14d193cdc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2') }} tmp -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql index c09c78954b90..1452594dfb23 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as CONFLICT_STREAM_SCALAR, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_SCALAR') }} as table_alias -- CONFLICT_STREAM_SCALAR +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql index 826e19497f0a..d889d1f350ce 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, cast(CONFLICT_STREAM_SCALAR as {{ dbt_utils.type_bigint() }}) as CONFLICT_STREAM_SCALAR, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('CONFLICT_STREAM_SCALAR_AB1') }} -- CONFLICT_STREAM_SCALAR +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql index 225410177ee0..5b6d06708f98 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('CONFLICT_STREAM_SCALAR_AB2') }} tmp -- CONFLICT_STREAM_SCALAR +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql index 2b9b9145d113..b74a533e30d4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -1,10 +1,19 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as DATE, {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as PARTITION, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} as table_alias -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql index b294f52bd063..6acd4a631460 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -1,10 +1,19 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, cast(DATE as {{ dbt_utils.type_string() }}) as DATE, cast(PARTITION as {{ type_json() }}) as PARTITION, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1') }} -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql deleted file mode 100644 index 2e20f30cde09..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3') }} tmp --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql index 18f29ba28605..d9183add3292 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, {{ json_extract_array('PARTITION', ['double_array_data'], ['double_array_data']) }} as DOUBLE_ARRAY_DATA, {{ json_extract_array('PARTITION', ['DATA'], ['DATA']) }} as DATA, {{ json_extract_array('PARTITION', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} as table_alias -where PARTITION is not null -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and PARTITION is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql index 798cdc9e347f..f610a8eb8bca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql @@ -1,11 +1,19 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, DOUBLE_ARRAY_DATA, DATA, {{ adapter.quote('column`_\'with""_quotes') }}, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1') }} -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql index 68df2747dfe5..8688fa6761c6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -10,4 +15,5 @@ select tmp.* from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2') }} tmp -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql index dfbff2614d4e..6a50aee02e79 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION', 'PARTITION', adapter.quote('column`_\'with""_quotes')) }} select _AIRBYTE_PARTITION_HASHID, {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} as table_alias -{{ cross_join_unnest('PARTITION', adapter.quote('column`_\'with""_quotes')) }} -where {{ adapter.quote('column`_\'with""_quotes') }} is not null -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +{{ cross_join_unnest('PARTITION', adapter.quote('column`_\'with""_quotes')) }} +where 1 = 1 +and {{ adapter.quote('column`_\'with""_quotes') }} is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql index 98f23739dc48..301b85b8e983 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_PARTITION_HASHID, cast(CURRENCY as {{ dbt_utils.type_string() }}) as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1') }} -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql index 47cae36ad121..b4f7cd0bdb61 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2') }} tmp -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql index ba79ad031f77..daa25f87285e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION', 'PARTITION', 'DATA') }} select _AIRBYTE_PARTITION_HASHID, {{ json_extract_scalar(unnested_column_value('DATA'), ['currency'], ['currency']) }} as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} as table_alias -{{ cross_join_unnest('PARTITION', 'DATA') }} -where DATA is not null -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +{{ cross_join_unnest('PARTITION', 'DATA') }} +where 1 = 1 +and DATA is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql index cc05da50b5da..81f8c127d424 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_PARTITION_HASHID, cast(CURRENCY as {{ dbt_utils.type_string() }}) as CURRENCY, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql index 6513fa3099fc..11dd0dc3fc02 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2') }} tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql index 6e51af8df4f2..281b8400bb15 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql @@ -1,12 +1,20 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION', 'PARTITION', 'DOUBLE_ARRAY_DATA') }} select _AIRBYTE_PARTITION_HASHID, {{ json_extract_scalar(unnested_column_value('DOUBLE_ARRAY_DATA'), ['id'], ['id']) }} as ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} as table_alias -{{ cross_join_unnest('PARTITION', 'DOUBLE_ARRAY_DATA') }} -where DOUBLE_ARRAY_DATA is not null -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +{{ cross_join_unnest('PARTITION', 'DOUBLE_ARRAY_DATA') }} +where 1 = 1 +and DOUBLE_ARRAY_DATA is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql index 2c37ca2f2a42..f036c8235e70 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_PARTITION_HASHID, cast(ID as {{ dbt_utils.type_string() }}) as ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1') }} -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql index 7c756d2b9cc9..143f944237e4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2') }} tmp -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql index 9820def18563..6692c6fcf348 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} as table_alias -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql index 1e1ebdd8fe21..b3926083d3ef 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, cast(DATE as {{ dbt_utils.type_string() }}) as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1') }} -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql index e033edcff94c..bd557dcba2d1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2') }} tmp -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql index 3fb9992e3ed6..f410de949a5f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql @@ -1,9 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as CHILDREN, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_UNNEST_ALIAS') }} as table_alias -- UNNEST_ALIAS +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql index 176d126943ef..15ef88171eae 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql @@ -1,9 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, CHILDREN, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('UNNEST_ALIAS_AB1') }} -- UNNEST_ALIAS +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql index 89c52952e230..8b4fd76705ce 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,6 @@ select tmp.* from {{ ref('UNNEST_ALIAS_AB2') }} tmp -- UNNEST_ALIAS +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql index 140ffe367d6f..3927a7ba55db 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema {{ unnest_cte('UNNEST_ALIAS', 'UNNEST_ALIAS', 'CHILDREN') }} select _AIRBYTE_UNNEST_ALIAS_HASHID, {{ json_extract_scalar(unnested_column_value('CHILDREN'), ['ab_id'], ['ab_id']) }} as AB_ID, {{ json_extract('', unnested_column_value('CHILDREN'), ['owner'], ['owner']) }} as OWNER, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('UNNEST_ALIAS') }} as table_alias -{{ cross_join_unnest('UNNEST_ALIAS', 'CHILDREN') }} -where CHILDREN is not null -- CHILDREN at unnest_alias/children +{{ cross_join_unnest('UNNEST_ALIAS', 'CHILDREN') }} +where 1 = 1 +and CHILDREN is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql index d3dac77c6240..f4726f1b470e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql @@ -1,10 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_UNNEST_ALIAS_HASHID, cast(AB_ID as {{ dbt_utils.type_bigint() }}) as AB_ID, cast(OWNER as {{ type_json() }}) as OWNER, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('UNNEST_ALIAS_CHILDREN_AB1') }} -- CHILDREN at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql index bfcc222e2e4f..57b65d2b40db 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +14,5 @@ select tmp.* from {{ ref('UNNEST_ALIAS_CHILDREN_AB2') }} tmp -- CHILDREN at unnest_alias/children +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql index b0f2a4bee14f..a80f261e0935 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql @@ -1,10 +1,18 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select _AIRBYTE_CHILDREN_HASHID, {{ json_extract_scalar('OWNER', ['owner_id'], ['owner_id']) }} as OWNER_ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('UNNEST_ALIAS_CHILDREN') }} as table_alias -where OWNER is not null -- OWNER at unnest_alias/children/owner +where 1 = 1 +and OWNER is not null diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql index 930146382f99..156bdcb32bce 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select _AIRBYTE_CHILDREN_HASHID, cast(OWNER_ID as {{ dbt_utils.type_bigint() }}) as OWNER_ID, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('UNNEST_ALIAS_CHILDREN_OWNER_AB1') }} -- OWNER at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql index e21dff47783c..07a3fe2b02c4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["nested-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "nested-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('UNNEST_ALIAS_CHILDREN_OWNER_AB2') }} tmp -- OWNER at unnest_alias/children/owner +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql index 4fddf6417250..1c660548d9ef 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION_NAMESPACE', '_AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} as table_alias -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql index 3586cbb1a476..d4ba14c99afb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -1,9 +1,17 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_string() }}) as ID, cast(DATE as {{ dbt_utils.type_string() }}) as DATE, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1') }} -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql index b75132fc0ce9..56780a4b7cf3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION_NAMESPACE", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -8,4 +13,5 @@ select tmp.* from {{ ref('SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2') }} tmp -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql similarity index 53% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql index 6bf3a89d196f..0908a9ca3ec4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql @@ -1,12 +1,22 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _AIRBYTE_UNIQUE_KEY, ID, DATE, PARTITION, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD') }} -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql new file mode 100644 index 000000000000..eaaa8ed6bc11 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql @@ -0,0 +1,19 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + ID, + CHILDREN, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_UNNEST_ALIAS_HASHID +from {{ ref('UNNEST_ALIAS_AB3') }} +-- UNNEST_ALIAS from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_UNNEST_ALIAS') }} +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql new file mode 100644 index 000000000000..a47501b4ad35 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -0,0 +1,108 @@ +{{ config( + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3') }} + -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} + where 1 = 1 + {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} +), +new_data_ids as ( + -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3') }} as inc_data on 1 = 0 + where _AIRBYTE_ACTIVE_ROW = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3')) }} from new_data + union all + select {{ dbt_utils.star(ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3') }} + -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY, + ID, + DATE, + PARTITION, + DATE as _AIRBYTE_START_AT, + lag(DATE) over ( + partition by ID + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(DATE) over ( + partition by ID + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + {{ dbt_utils.surrogate_key([ + '_AIRBYTE_UNIQUE_KEY', + '_AIRBYTE_START_AT', + '_AIRBYTE_EMITTED_AT' + ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + DATE, + PARTITION, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql index 258221562b2d..9f76e65e0fcd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID from {{ ref('CONFLICT_STREAM_ARRAY_AB3') }} -- CONFLICT_STREAM_ARRAY from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_ARRAY') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql index c35f1fb8aaaa..713b00119ec0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_NAME_HASHID from {{ ref('CONFLICT_STREAM_NAME_AB3') }} -- CONFLICT_STREAM_NAME from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_NAME') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql index 5d4ac2bec638..b28901245add 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3') }} -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name from {{ ref('CONFLICT_STREAM_NAME') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql index 1cbd75908309..9e873bdf985d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, GROUPS, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3') }} -- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql index 13afbef4392a..57ea8746c22d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID from {{ ref('CONFLICT_STREAM_SCALAR_AB3') }} -- CONFLICT_STREAM_SCALAR from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_CONFLICT_STREAM_SCALAR') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql index 456f08e6bb2d..05cd2e777323 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql @@ -1,12 +1,20 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, DOUBLE_ARRAY_DATA, DATA, {{ adapter.quote('column`_\'with""_quotes') }}, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_PARTITION_HASHID from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3') }} -- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql index cb8d1021e10e..cfcb9c78f631 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_PARTITION_HASHID, CURRENCY, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_COLUMN___WITH__QUOTES_HASHID from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3') }} -- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql index 97b9fb2d61fb..fb8b719df25e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_PARTITION_HASHID, CURRENCY, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_DATA_HASHID from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3') }} -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql index ce4af8494123..4282011cdce6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_PARTITION_HASHID, ID, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3') }} -- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql deleted file mode 100644 index 5ab034b5cb5a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ /dev/null @@ -1,21 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - DATE, - PARTITION, - DATE as _AIRBYTE_START_AT, - lag(DATE) over ( - partition by ID - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( - partition by ID - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID -from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB4') }} --- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql index fdce34e55683..e8419fbe61a8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, DATE, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID from {{ ref('NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3') }} -- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql deleted file mode 100644 index 0eca0476eb43..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS.sql +++ /dev/null @@ -1,10 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- Final base SQL model -select - ID, - CHILDREN, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_UNNEST_ALIAS_HASHID -from {{ ref('UNNEST_ALIAS_AB3') }} --- UNNEST_ALIAS from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_UNNEST_ALIAS') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql index 6a76c5c0ed8a..d94783e6631d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql @@ -1,11 +1,19 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_UNNEST_ALIAS_HASHID, AB_ID, OWNER, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_CHILDREN_HASHID from {{ ref('UNNEST_ALIAS_CHILDREN_AB3') }} -- CHILDREN at unnest_alias/children from {{ ref('UNNEST_ALIAS') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql index 26769a1920c7..639a820f74be 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["nested"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "nested" ] +) }} -- Final base SQL model select _AIRBYTE_CHILDREN_HASHID, OWNER_ID, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_OWNER_HASHID from {{ ref('UNNEST_ALIAS_CHILDREN_OWNER_AB3') }} -- OWNER at unnest_alias/children/owner from {{ ref('UNNEST_ALIAS_CHILDREN') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql index ba2beaf304f5..67e8f286cd08 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -1,10 +1,18 @@ -{{ config(schema="TEST_NORMALIZATION_NAMESPACE", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION_NAMESPACE", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, DATE, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID from {{ ref('SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3') }} -- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES from {{ source('TEST_NORMALIZATION_NAMESPACE', '_AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES') }} +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql similarity index 60% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql index 353e4f8c3117..adb912d6a122 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -9,4 +14,6 @@ select tmp.* from {{ ref('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2') }} tmp -- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql new file mode 100644 index 000000000000..e18e6a208a2c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + get_path(parse_json(_airbyte_data), '"conflict_stream_array"') as CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY as table_alias +-- CONFLICT_STREAM_ARRAY +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql new file mode 100644 index 000000000000..97e976fd6462 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB1" +-- CONFLICT_STREAM_ARRAY +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql new file mode 100644 index 000000000000..66a710356b32 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_ARRAY as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB2" tmp +-- CONFLICT_STREAM_ARRAY +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql new file mode 100644 index 000000000000..1a41bd6a787d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB1.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + + get_path(parse_json(table_alias._airbyte_data), '"conflict_stream_name"') + as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME as table_alias +-- CONFLICT_STREAM_NAME +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql new file mode 100644 index 000000000000..bf735e634dcb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(CONFLICT_STREAM_NAME as + variant +) as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB1" +-- CONFLICT_STREAM_NAME +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql new file mode 100644 index 000000000000..3fe0514dbf9f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB2" tmp +-- CONFLICT_STREAM_NAME +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql new file mode 100644 index 000000000000..e3ef93a18792 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + + get_path(parse_json(table_alias.CONFLICT_STREAM_NAME), '"conflict_stream_name"') + as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as table_alias +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql new file mode 100644 index 000000000000..e9b6b9804fd9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + cast(CONFLICT_STREAM_NAME as + variant +) as CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql new file mode 100644 index 000000000000..0616d66795ea --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_HASHID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_NAME as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql new file mode 100644 index 000000000000..3e793970e09c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1.sql @@ -0,0 +1,15 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + to_varchar(get_path(parse_json(CONFLICT_STREAM_NAME), '"groups"')) as GROUPS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as table_alias +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 +and CONFLICT_STREAM_NAME is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql new file mode 100644 index 000000000000..a405f7059b95 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + cast(GROUPS as + varchar +) as GROUPS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB1" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql new file mode 100644 index 000000000000..26ab7e066acf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID as + varchar +), '') || '-' || coalesce(cast(GROUPS as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB2" tmp +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql new file mode 100644 index 000000000000..d61b09ff806a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"conflict_stream_scalar"')) as CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR as table_alias +-- CONFLICT_STREAM_SCALAR +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql new file mode 100644 index 000000000000..703883993977 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(CONFLICT_STREAM_SCALAR as + bigint +) as CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB1" +-- CONFLICT_STREAM_SCALAR +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql new file mode 100644 index 000000000000..00e7c7886c89 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CONFLICT_STREAM_SCALAR as + varchar +), '') as + varchar +)) as _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB2" tmp +-- CONFLICT_STREAM_SCALAR +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql new file mode 100644 index 000000000000..6e7e79743d05 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + + get_path(parse_json(table_alias._airbyte_data), '"partition"') + as PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES as table_alias +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql new file mode 100644 index 000000000000..eb9c920cc525 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -0,0 +1,22 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + cast(PARTITION as + variant +) as PARTITION, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB1" +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql new file mode 100644 index 000000000000..7b11ebe9ba64 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + get_path(parse_json(PARTITION), '"double_array_data"') as DOUBLE_ARRAY_DATA, + get_path(parse_json(PARTITION), '"DATA"') as DATA, + get_path(parse_json(PARTITION), '"column`_''with""_quotes"') as "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as table_alias +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 +and PARTITION is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql new file mode 100644 index 000000000000..8c9edac47baf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + DOUBLE_ARRAY_DATA, + DATA, + "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1" +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql new file mode 100644 index 000000000000..d659cf82cab8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3.sql @@ -0,0 +1,21 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID as + varchar +), '') || '-' || coalesce(cast(DOUBLE_ARRAY_DATA as + varchar +), '') || '-' || coalesce(cast(DATA as + varchar +), '') || '-' || coalesce(cast("column`_'with""_quotes" as + varchar +), '') as + varchar +)) as _AIRBYTE_PARTITION_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2" tmp +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql new file mode 100644 index 000000000000..17a36065ce62 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json("column`_'with""_quotes".value), '"currency"')) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +cross join table(flatten("column`_'with""_quotes")) as "column`_'with""_quotes" +where 1 = 1 +and "column`_'with""_quotes" is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql new file mode 100644 index 000000000000..de013bb4fb92 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(CURRENCY as + varchar +) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB1" +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql new file mode 100644 index 000000000000..1996258464ce --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') as + varchar +)) as _AIRBYTE_COLUMN___WITH__QUOTES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB2" tmp +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql new file mode 100644 index 000000000000..fbd05545c768 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json(DATA.value), '"currency"')) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +cross join table(flatten(DATA)) as DATA +where 1 = 1 +and DATA is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql new file mode 100644 index 000000000000..bdf9cd2b386c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(CURRENCY as + varchar +) as CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB1" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql new file mode 100644 index 000000000000..c4a6deea96b7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') as + varchar +)) as _AIRBYTE_DATA_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB2" tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql new file mode 100644 index 000000000000..3013309f1f90 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_PARTITION_HASHID, + to_varchar(get_path(parse_json(DOUBLE_ARRAY_DATA.value), '"id"')) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as table_alias +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +cross join table(flatten(DOUBLE_ARRAY_DATA)) as DOUBLE_ARRAY_DATA +where 1 = 1 +and DOUBLE_ARRAY_DATA is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql new file mode 100644 index 000000000000..5c1e8b226a6a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_PARTITION_HASHID, + cast(ID as + varchar +) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1" +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql new file mode 100644 index 000000000000..662055a5ee44 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_PARTITION_HASHID as + varchar +), '') || '-' || coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2" tmp +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql new file mode 100644 index 000000000000..44ffce39889f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql new file mode 100644 index 000000000000..437075adbab4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql new file mode 100644 index 000000000000..70f3a3546307 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') as + varchar +)) as _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql new file mode 100644 index 000000000000..3c6d0a2e24d2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB1.sql @@ -0,0 +1,15 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + get_path(parse_json(_airbyte_data), '"children"') as CHILDREN, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS as table_alias +-- UNNEST_ALIAS +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql new file mode 100644 index 000000000000..520bc1d1d7ec --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB2.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + CHILDREN, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB1" +-- UNNEST_ALIAS +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql new file mode 100644 index 000000000000..753d624ac210 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_AB3.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CHILDREN as + varchar +), '') as + varchar +)) as _AIRBYTE_UNNEST_ALIAS_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_AB2" tmp +-- UNNEST_ALIAS +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql new file mode 100644 index 000000000000..a4717707483e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB1.sql @@ -0,0 +1,20 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _AIRBYTE_UNNEST_ALIAS_HASHID, + to_varchar(get_path(parse_json(CHILDREN.value), '"ab_id"')) as AB_ID, + + get_path(parse_json(CHILDREN.value), '"owner"') + as OWNER, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as table_alias +-- CHILDREN at unnest_alias/children +cross join table(flatten(CHILDREN)) as CHILDREN +where 1 = 1 +and CHILDREN is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql new file mode 100644 index 000000000000..b0fd97c240a0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB2.sql @@ -0,0 +1,19 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_UNNEST_ALIAS_HASHID, + cast(AB_ID as + bigint +) as AB_ID, + cast(OWNER as + variant +) as OWNER, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB1" +-- CHILDREN at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql new file mode 100644 index 000000000000..88b1593cc4d0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_AB3.sql @@ -0,0 +1,19 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_UNNEST_ALIAS_HASHID as + varchar +), '') || '-' || coalesce(cast(AB_ID as + varchar +), '') || '-' || coalesce(cast(OWNER as + varchar +), '') as + varchar +)) as _AIRBYTE_CHILDREN_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB2" tmp +-- CHILDREN at unnest_alias/children +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql new file mode 100644 index 000000000000..806d713ba2f3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB1.sql @@ -0,0 +1,15 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _AIRBYTE_CHILDREN_HASHID, + to_varchar(get_path(parse_json(OWNER), '"owner_id"')) as OWNER_ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as table_alias +-- OWNER at unnest_alias/children/owner +where 1 = 1 +and OWNER is not null + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql new file mode 100644 index 000000000000..4e3f70fc2e82 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _AIRBYTE_CHILDREN_HASHID, + cast(OWNER_ID as + bigint +) as OWNER_ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB1" +-- OWNER at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql new file mode 100644 index 000000000000..79189e35ffbe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(_AIRBYTE_CHILDREN_HASHID as + varchar +), '') || '-' || coalesce(cast(OWNER_ID as + varchar +), '') as + varchar +)) as _AIRBYTE_OWNER_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB2" tmp +-- OWNER at unnest_alias/children/owner +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql new file mode 100644 index 000000000000..dc9d26c10728 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES as table_alias +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql new file mode 100644 index 000000000000..ac66f7e51acd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + varchar +) as ID, + cast(DATE as + varchar +) as DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB1" +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql new file mode 100644 index 000000000000..0becb20f6e1b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_ctes/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -0,0 +1,17 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') as + varchar +)) as _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB2" tmp +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql new file mode 100644 index 000000000000..38b63ebc54d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","ID" = DBT_INTERNAL_SOURCE."ID","DATE" = DBT_INTERNAL_SOURCE."DATE","PARTITION" = DBT_INTERNAL_SOURCE."PARTITION","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "ID", "DATE", "PARTITION", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "ID", "DATE", "PARTITION", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql new file mode 100644 index 000000000000..6938df462140 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/UNNEST_ALIAS.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_AB_ID = DBT_INTERNAL_DEST._AIRBYTE_AB_ID + + + + when matched then update set + "ID" = DBT_INTERNAL_SOURCE."ID","CHILDREN" = DBT_INTERNAL_SOURCE."CHILDREN","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_UNNEST_ALIAS_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNNEST_ALIAS_HASHID" + + + when not matched then insert + ("ID", "CHILDREN", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_UNNEST_ALIAS_HASHID") + values + ("ID", "CHILDREN", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_UNNEST_ALIAS_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql new file mode 100644 index 000000000000..cbca5b47ca42 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY_SCD = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY_SCD + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","_AIRBYTE_UNIQUE_KEY_SCD" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY_SCD","ID" = DBT_INTERNAL_SOURCE."ID","DATE" = DBT_INTERNAL_SOURCE."DATE","PARTITION" = DBT_INTERNAL_SOURCE."PARTITION","_AIRBYTE_START_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_START_AT","_AIRBYTE_END_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_END_AT","_AIRBYTE_ACTIVE_ROW" = DBT_INTERNAL_SOURCE."_AIRBYTE_ACTIVE_ROW","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "DATE", "PARTITION", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "DATE", "PARTITION", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql new file mode 100644 index 000000000000..a366f2bba8cc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_ARRAY.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" as + (select * from( + +-- Final base SQL model +select + ID, + CONFLICT_STREAM_ARRAY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_ARRAY_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY_AB3" +-- CONFLICT_STREAM_ARRAY from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_ARRAY +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_ARRAY" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql new file mode 100644 index 000000000000..a20080a7f7f4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" as + (select * from( + +-- Final base SQL model +select + ID, + CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_AB3" +-- CONFLICT_STREAM_NAME from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_NAME +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql new file mode 100644 index 000000000000..4d69fa730172 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_CONFLICT_STREAM_NAME_HASHID, + CONFLICT_STREAM_NAME, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql new file mode 100644 index 000000000000..97d25955385f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_CONFLICT_STREAM_NAME_2_HASHID, + GROUPS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_NAME_3_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_AB3" +-- CONFLICT_STREAM_NAME at conflict_stream_name/conflict_stream_name/conflict_stream_name from "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME_CONFLICT_STREAM_NAME" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql new file mode 100644 index 000000000000..efc7cbe9dce6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/CONFLICT_STREAM_SCALAR.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" as + (select * from( + +-- Final base SQL model +select + ID, + CONFLICT_STREAM_SCALAR, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CONFLICT_STREAM_SCALAR_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR_AB3" +-- CONFLICT_STREAM_SCALAR from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_CONFLICT_STREAM_SCALAR +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."CONFLICT_STREAM_SCALAR" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql new file mode 100644 index 000000000000..75228a851232 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION.sql @@ -0,0 +1,21 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + DOUBLE_ARRAY_DATA, + DATA, + "column`_'with""_quotes", + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_PARTITION_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3" +-- PARTITION at nested_stream_with_complex_columns_resulting_into_long_names/partition from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql new file mode 100644 index 000000000000..1f2adeefe899 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_COLUMN___WITH__QUOTES_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES_AB3" +-- COLUMN___WITH__QUOTES at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_COLUMN___WITH__QUOTES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql new file mode 100644 index 000000000000..cfc1ce750875 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + CURRENCY, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DATA_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA_AB3" +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql new file mode 100644 index 000000000000..4bdef077e4b3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_PARTITION_HASHID, + ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DOUBLE_ARRAY_DATA_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3" +-- DOUBLE_ARRAY_DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql new file mode 100644 index 000000000000..a470f14eceac --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" as + (select * from( + +-- Final base SQL model +select + ID, + DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" +-- NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."NON_NESTED_STREAM_WITHOUT_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql new file mode 100644 index 000000000000..0657b14de313 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN.sql @@ -0,0 +1,20 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNNEST_ALIAS_HASHID, + AB_ID, + OWNER, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_CHILDREN_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_AB3" +-- CHILDREN at unnest_alias/children from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql new file mode 100644 index 000000000000..652a9bd48931 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION/UNNEST_ALIAS_CHILDREN_OWNER.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_CHILDREN_HASHID, + OWNER_ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_OWNER_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER_AB3" +-- OWNER at unnest_alias/children/owner from "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN" +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."UNNEST_ALIAS_CHILDREN_OWNER" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql new file mode 100644 index 000000000000..057fea2c9ee7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_tables/TEST_NORMALIZATION_NAMESPACE/SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES.sql @@ -0,0 +1,19 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" as + (select * from( + +-- Final base SQL model +select + ID, + DATE, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_HASHID +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES_AB3" +-- SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES from "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE._AIRBYTE_RAW_SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES +where 1 = 1 + ) order by (_AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION_NAMESPACE."SIMPLE_STREAM_WITH_NAMESPACE_RESULTING_INTO_LONG_NAMES" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql new file mode 100644 index 000000000000..b224c3ada40e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/second_output/airbyte_views/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3.sql @@ -0,0 +1,20 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(PARTITION as + varchar +), '') as + varchar +)) as _AIRBYTE_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_AB2" tmp +-- NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql deleted file mode 100644 index c52930b6182a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB4" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" tmp --- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql deleted file mode 100644 index 6205e9e1fbd1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB4" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB3" tmp --- DEDUP_EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql deleted file mode 100644 index 6b378a26afcf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql +++ /dev/null @@ -1,13 +0,0 @@ - - create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB4" as ( - --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_POS_DEDUP_CDCX_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" tmp --- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX - ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql deleted file mode 100644 index e083558a535d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql +++ /dev/null @@ -1,28 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" as - ( --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - NAME, - "column`_'with""_quotes", - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB4" --- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql deleted file mode 100644 index 87d94b17d25c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ /dev/null @@ -1,34 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" as - ( --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - "HKD@spéçiäl & characters", - HKD_SPECIAL___CHARACTERS, - NZD, - USD, - DATE as _AIRBYTE_START_AT, - lag(DATE) over ( - partition by ID, CURRENCY, cast(NZD as - varchar -) - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( - partition by ID, CURRENCY, cast(NZD as - varchar -) - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB4" --- DEDUP_EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql deleted file mode 100644 index 292013c109cb..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql +++ /dev/null @@ -1,28 +0,0 @@ - - - create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" as - ( --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB4" --- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX -where _airbyte_row_num = 1 - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql index 4ddfa66b4825..8d5282887fb0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql @@ -5,11 +5,14 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, - to_varchar(get_path(parse_json(_airbyte_data), '"column`_''with""_quotes"')) as "column`_'with""_quotes", to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED as table_alias -- DEDUP_CDC_EXCLUDED +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql similarity index 81% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql index dd52fb86ff3f..57a50d43e5bc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql @@ -9,9 +9,6 @@ select cast(NAME as varchar ) as NAME, - cast("column`_'with""_quotes" as - varchar -) as "column`_'with""_quotes", cast(_AB_CDC_LSN as float ) as _AB_CDC_LSN, @@ -21,7 +18,11 @@ select cast(_AB_CDC_DELETED_AT as float ) as _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" -- DEDUP_CDC_EXCLUDED +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql index 124336cd315d..ee207b768a20 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql @@ -11,7 +11,11 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE as table_alias -- DEDUP_EXCHANGE_RATE +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql index 7b76d03348ee..66b09ec2508e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql @@ -33,7 +33,11 @@ select cast(USD as float ) as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" -- DEDUP_EXCHANGE_RATE +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql index 427d37de5377..33e07a844a1c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql @@ -11,7 +11,11 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE as table_alias -- EXCHANGE_RATE +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql similarity index 91% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql index d9459cd75f1b..f5ef6e473984 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql @@ -33,7 +33,11 @@ select cast(USD as float ) as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" -- EXCHANGE_RATE +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql index c69cb6fc10b9..c1f0840c3677 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql @@ -25,4 +25,6 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" tmp -- EXCHANGE_RATE +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql similarity index 86% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql index fb06f48abfe5..93d03ade5063 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql @@ -9,7 +9,10 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_log_pos"')) as _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX as table_alias -- POS_DEDUP_CDCX +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql similarity index 82% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql index 0bd1e24eae02..82b6ca1c2619 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql @@ -21,7 +21,10 @@ select cast(_AB_CDC_LOG_POS as float ) as _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" -- POS_DEDUP_CDCX +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql new file mode 100644 index 000000000000..123971c4e5dc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED as table_alias +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql new file mode 100644 index 000000000000..35438f37c247 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql similarity index 53% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql index a530c9d73be3..08e7e20215a1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql @@ -1,19 +1,25 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED" as - ( + (select * from( + -- Final base SQL model select + _AIRBYTE_UNIQUE_KEY, ID, NAME, - "column`_'with""_quotes", _AB_CDC_LSN, _AB_CDC_UPDATED_AT, _AB_CDC_DELETED_AT, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID from "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" -- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED -where _airbyte_active_row = 1 +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql similarity index 56% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql index 89d7840d3a68..1a9c41c14890 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql @@ -1,9 +1,11 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" as - ( + (select * from( + -- Final base SQL model select + _AIRBYTE_UNIQUE_KEY, ID, CURRENCY, DATE, @@ -12,10 +14,15 @@ select HKD_SPECIAL___CHARACTERS, NZD, USD, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID from "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" -- DEDUP_EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE -where _airbyte_active_row = 1 +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql similarity index 62% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql index b669de625a2b..3420de9fa251 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -1,7 +1,8 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as - ( + (select * from( + -- Final base SQL model select ID, @@ -12,9 +13,14 @@ select HKD_SPECIAL___CHARACTERS, NZD, USD, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_EXCHANGE_RATE_HASHID from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB3" -- EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE +where 1 = 1 + + ) order by (_AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" cluster by (_AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..f5c3fb018edc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,21 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" +-- RENAMED_DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..343251eb8988 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,93 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" + -- DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc + ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as + varchar +), cast(_AB_CDC_UPDATED_AT as + varchar +) + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql new file mode 100644 index 000000000000..f1a29aefb339 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -0,0 +1,99 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB3" + -- DEDUP_EXCHANGE_RATE from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') || '-' || coalesce(cast(NZD as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + "HKD@spéçiäl & characters", + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + DATE as _AIRBYTE_START_AT, + lag(DATE) over ( + partition by ID, CURRENCY, cast(NZD as + varchar +) + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(DATE) over ( + partition by ID, CURRENCY, cast(NZD as + varchar +) + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + "HKD@spéçiäl & characters", + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..c171a978fe39 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,77 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB3" + -- RENAMED_DEDUP_CDC_EXCLUDED from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql similarity index 54% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql index 8892c9c7bda7..198a2627ab68 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql @@ -1,19 +1,25 @@ create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" as - ( + (select * from( + -- Final base SQL model select + _AIRBYTE_UNIQUE_KEY, ID, NAME, _AB_CDC_LSN, _AB_CDC_UPDATED_AT, _AB_CDC_DELETED_AT, _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_POS_DEDUP_CDCX_HASHID from "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" -- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX -where _airbyte_active_row = 1 +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) ); - \ No newline at end of file + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql new file mode 100644 index 000000000000..24b58ed5bf4d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql @@ -0,0 +1,99 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" + -- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as + varchar +), cast(_AB_CDC_UPDATED_AT as + varchar +), cast(_AB_CDC_LOG_POS as + varchar +) + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql similarity index 90% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql index a4b8a3aedf2f..522991af25cd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql @@ -7,8 +7,6 @@ select varchar ), '') || '-' || coalesce(cast(NAME as varchar -), '') || '-' || coalesce(cast("column`_'with""_quotes" as - varchar ), '') || '-' || coalesce(cast(_AB_CDC_LSN as varchar ), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as @@ -21,4 +19,6 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" tmp -- DEDUP_CDC_EXCLUDED +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql index 9049fa459fe0..0b361061abf3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql @@ -25,4 +25,6 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" tmp -- DEDUP_EXCHANGE_RATE +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql similarity index 98% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql index 5b8827c33977..c1189b3c1dfd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/final/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql @@ -21,4 +21,5 @@ select tmp.* from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" tmp -- POS_DEDUP_CDCX +where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql new file mode 100644 index 000000000000..624b8d286061 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" tmp +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql deleted file mode 100644 index b5ed6a14b8ad..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/dbt_data_tests_tmp/tmp_test_data_check_row_counts.sql +++ /dev/null @@ -1,43 +0,0 @@ -with table_row_counts as ( - select distinct count(*) as row_count, 10 as expected_count - from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_EXCHANGE_RATE') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('EXCHANGE_RATE') }} - -union all - - select distinct count(*) as row_count, 10 as expected_count - from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} -union all - select distinct count(*) as row_count, 10 as expected_count - from {{ ref('DEDUP_EXCHANGE_RATE_SCD') }} -union all - select distinct count(*) as row_count, 5 as expected_count - from {{ ref('DEDUP_EXCHANGE_RATE') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('DEDUP_CDC_EXCLUDED_SCD') }} -union all - select distinct count(*) as row_count, 4 as expected_count - from {{ ref('DEDUP_CDC_EXCLUDED') }} - -union all - - select distinct count(*) as row_count, 8 as expected_count - from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} -union all - select distinct count(*) as row_count, 8 as expected_count - from {{ ref('POS_DEDUP_CDCX_SCD') }} -union all - select distinct count(*) as row_count, 3 as expected_count - from {{ ref('POS_DEDUP_CDCX') }} -) -select * -from table_row_counts -where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql index f4ad52f55e61..d1960bc9d6f6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql @@ -1,13 +1,21 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as NAME, - {{ json_extract_scalar('_airbyte_data', ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _AB_CDC_LSN, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _AB_CDC_UPDATED_AT, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} as table_alias -- DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql index 39a6396a3344..53573e0731e8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql @@ -1,13 +1,21 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, cast(NAME as {{ dbt_utils.type_string() }}) as NAME, - cast({{ adapter.quote('column`_\'with""_quotes') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('column`_\'with""_quotes') }}, cast(_AB_CDC_LSN as {{ dbt_utils.type_float() }}) as _AB_CDC_LSN, cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_UPDATED_AT, cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('DEDUP_CDC_EXCLUDED_AB1') }} -- DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql deleted file mode 100644 index fa5fa68a458c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} tmp --- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql index bdc1a517763e..52686abcae76 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, @@ -9,7 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as HKD_SPECIAL___CHARACTERS, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} as table_alias -- DEDUP_EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql index 1851076c48c9..66aad2b9bd12 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, @@ -17,7 +22,11 @@ select cast(HKD_SPECIAL___CHARACTERS as {{ dbt_utils.type_string() }}) as HKD_SPECIAL___CHARACTERS, cast(NZD as {{ dbt_utils.type_float() }}) as NZD, cast(USD as {{ dbt_utils.type_float() }}) as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('DEDUP_EXCHANGE_RATE_AB1') }} -- DEDUP_EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql deleted file mode 100644 index 6cae7e097fe8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from {{ ref('DEDUP_EXCHANGE_RATE_AB3') }} tmp --- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql index fffa51f8b038..d2d59c06a80b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, @@ -9,7 +14,11 @@ select {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as HKD_SPECIAL___CHARACTERS, {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as NZD, {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_EXCHANGE_RATE') }} as table_alias -- EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql index 473aeec716da..d61195d2eb84 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, @@ -17,7 +22,11 @@ select cast(HKD_SPECIAL___CHARACTERS as {{ dbt_utils.type_string() }}) as HKD_SPECIAL___CHARACTERS, cast(NZD as {{ dbt_utils.type_float() }}) as NZD, cast(USD as {{ dbt_utils.type_float() }}) as USD, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('EXCHANGE_RATE_AB1') }} -- EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql index 239c8e6caf12..db5f1117000f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +19,6 @@ select tmp.* from {{ ref('EXCHANGE_RATE_AB2') }} tmp -- EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql index b26e74409945..cae6bceaf73d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, @@ -7,7 +12,10 @@ select {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _AB_CDC_UPDATED_AT, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _AB_CDC_DELETED_AT, {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_log_pos'], ['_ab_cdc_log_pos']) }} as _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} as table_alias -- POS_DEDUP_CDCX +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql index 361817894307..9e48e834cd99 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type select cast(ID as {{ dbt_utils.type_bigint() }}) as ID, @@ -7,7 +12,10 @@ select cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_UPDATED_AT, cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_float() }}) as _AB_CDC_DELETED_AT, cast(_AB_CDC_LOG_POS as {{ dbt_utils.type_float() }}) as _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT from {{ ref('POS_DEDUP_CDCX_AB1') }} -- POS_DEDUP_CDCX +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql deleted file mode 100644 index 7e454448034e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB4.sql +++ /dev/null @@ -1,11 +0,0 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} --- SQL model to prepare for deduplicating records based on the hash record column -select - row_number() over ( - partition by _AIRBYTE_POS_DEDUP_CDCX_HASHID - order by _AIRBYTE_EMITTED_AT asc - ) as _AIRBYTE_ROW_NUM, - tmp.* -from {{ ref('POS_DEDUP_CDCX_AB3') }} tmp --- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql new file mode 100644 index 000000000000..de7672d5c9df --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql @@ -0,0 +1,17 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT +from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} as table_alias +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql new file mode 100644 index 000000000000..8597d9cbf0fa --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql @@ -0,0 +1,17 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as {{ dbt_utils.type_bigint() }}) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT +from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB1') }} +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..6aa5c273499d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,24 @@ +{{ config( + cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID +from {{ ref('DEDUP_CDC_EXCLUDED_SCD') }} +-- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql similarity index 51% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql index 531a13cf7b5a..e835ffe0c1bb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql @@ -1,6 +1,12 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _AIRBYTE_UNIQUE_KEY, ID, CURRENCY, DATE, @@ -9,9 +15,13 @@ select HKD_SPECIAL___CHARACTERS, NZD, USD, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID from {{ ref('DEDUP_EXCHANGE_RATE_SCD') }} -- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql similarity index 52% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql index cfaae19243b7..533791bfb80e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -1,4 +1,9 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select ID, @@ -9,8 +14,12 @@ select HKD_SPECIAL___CHARACTERS, NZD, USD, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_EXCHANGE_RATE_HASHID from {{ ref('EXCHANGE_RATE_AB3') }} -- EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_EXCHANGE_RATE') }} +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..6c761d93c54f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,20 @@ +{{ config( + cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID +from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_SCD') }} +-- RENAMED_DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..6fcbf7b8e63d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,112 @@ +{{ config( + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} + -- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} + where 1 = 1 + {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} +), +new_data_ids as ( + -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('DEDUP_CDC_EXCLUDED_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} as inc_data on 1 = 0 + where _AIRBYTE_ACTIVE_ROW = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('DEDUP_CDC_EXCLUDED_AB3')) }} from new_data + union all + select {{ dbt_utils.star(ref('DEDUP_CDC_EXCLUDED_AB3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('DEDUP_CDC_EXCLUDED_AB3') }} + -- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc + ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_string() }}), cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_string() }}) + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + {{ dbt_utils.surrogate_key([ + '_AIRBYTE_UNIQUE_KEY', + '_AIRBYTE_START_AT', + '_AIRBYTE_EMITTED_AT', '_AB_CDC_DELETED_AT', '_AB_CDC_UPDATED_AT' + ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql new file mode 100644 index 000000000000..060a3ccb799a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -0,0 +1,122 @@ +{{ config( + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('DEDUP_EXCHANGE_RATE_AB3') }} + -- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} + where 1 = 1 + {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} +), +new_data_ids as ( + -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'ID', + 'CURRENCY', + 'NZD', + ]) }} as _AIRBYTE_UNIQUE_KEY + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('DEDUP_EXCHANGE_RATE_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('DEDUP_EXCHANGE_RATE_AB3') }} as inc_data on 1 = 0 + where _AIRBYTE_ACTIVE_ROW = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('DEDUP_EXCHANGE_RATE_AB3')) }} from new_data + union all + select {{ dbt_utils.star(ref('DEDUP_EXCHANGE_RATE_AB3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('DEDUP_EXCHANGE_RATE_AB3') }} + -- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'ID', + 'CURRENCY', + 'NZD', + ]) }} as _AIRBYTE_UNIQUE_KEY, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + DATE as _AIRBYTE_START_AT, + lag(DATE) over ( + partition by ID, CURRENCY, cast(NZD as {{ dbt_utils.type_string() }}) + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(DATE) over ( + partition by ID, CURRENCY, cast(NZD as {{ dbt_utils.type_string() }}) + order by + DATE is null asc, + DATE desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + {{ dbt_utils.surrogate_key([ + '_AIRBYTE_UNIQUE_KEY', + '_AIRBYTE_START_AT', + '_AIRBYTE_EMITTED_AT' + ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + CURRENCY, + DATE, + TIMESTAMP_COL, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + HKD_SPECIAL___CHARACTERS, + NZD, + USD, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..c212769bc05f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,104 @@ +{{ config( + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3') }} + -- RENAMED_DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} + where 1 = 1 + {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} +), +new_data_ids as ( + -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3') }} as inc_data on 1 = 0 + where _AIRBYTE_ACTIVE_ROW = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3')) }} from new_data + union all + select {{ dbt_utils.star(ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB3') }} + -- RENAMED_DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY, + ID, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc + ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + {{ dbt_utils.surrogate_key([ + '_AIRBYTE_UNIQUE_KEY', + '_AIRBYTE_START_AT', + '_AIRBYTE_EMITTED_AT' + ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql deleted file mode 100644 index 8335a889e120..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql +++ /dev/null @@ -1,15 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- Final base SQL model -select - ID, - NAME, - {{ adapter.quote('column`_\'with""_quotes') }}, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from {{ ref('DEDUP_CDC_EXCLUDED_SCD') }} --- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} -where _airbyte_active_row = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql deleted file mode 100644 index bc77ae8e0646..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - NAME, - {{ adapter.quote('column`_\'with""_quotes') }}, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID -from {{ ref('DEDUP_CDC_EXCLUDED_AB4') }} --- DEDUP_CDC_EXCLUDED from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_CDC_EXCLUDED') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql deleted file mode 100644 index f5f0c288d57f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ /dev/null @@ -1,26 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - CURRENCY, - DATE, - TIMESTAMP_COL, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - HKD_SPECIAL___CHARACTERS, - NZD, - USD, - DATE as _AIRBYTE_START_AT, - lag(DATE) over ( - partition by ID, CURRENCY, cast(NZD as {{ dbt_utils.type_string() }}) - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(DATE) over ( - partition by ID, CURRENCY, cast(NZD as {{ dbt_utils.type_string() }}) - order by DATE is null asc, DATE desc, _AIRBYTE_EMITTED_AT desc - ) is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID -from {{ ref('DEDUP_EXCHANGE_RATE_AB4') }} --- DEDUP_EXCHANGE_RATE from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_DEDUP_EXCHANGE_RATE') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql index 27120ccce874..1ab6634ee9b5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql @@ -1,15 +1,24 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} +{{ config( + cluster_by = ["_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} -- Final base SQL model select + _AIRBYTE_UNIQUE_KEY, ID, NAME, _AB_CDC_LSN, _AB_CDC_UPDATED_AT, _AB_CDC_DELETED_AT, _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, _AIRBYTE_POS_DEDUP_CDCX_HASHID from {{ ref('POS_DEDUP_CDCX_SCD') }} -- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} -where _airbyte_active_row = 1 +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql deleted file mode 100644 index 0ab8c0427373..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config(schema="TEST_NORMALIZATION", tags=["top-level"]) }} --- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key -select - ID, - NAME, - _AB_CDC_LSN, - _AB_CDC_UPDATED_AT, - _AB_CDC_DELETED_AT, - _AB_CDC_LOG_POS, - _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, - lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc - ) as _AIRBYTE_END_AT, - case when lag(_AIRBYTE_EMITTED_AT) over ( - partition by ID - order by _AIRBYTE_EMITTED_AT is null asc, _AIRBYTE_EMITTED_AT desc, _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc - ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, - _AIRBYTE_EMITTED_AT, - _AIRBYTE_POS_DEDUP_CDCX_HASHID -from {{ ref('POS_DEDUP_CDCX_AB4') }} --- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} -where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql new file mode 100644 index 000000000000..257fa09afe15 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql @@ -0,0 +1,114 @@ +{{ config( + cluster_by = ["_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_EMITTED_AT"], + unique_key = "_AIRBYTE_UNIQUE_KEY_SCD", + schema = "TEST_NORMALIZATION", + tags = [ "top-level" ] +) }} +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('POS_DEDUP_CDCX_AB3') }} + -- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} + where 1 = 1 + {{ incremental_clause('_AIRBYTE_EMITTED_AT') }} +), +new_data_ids as ( + -- build a subset of _AIRBYTE_UNIQUE_KEY from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY + from new_data +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('POS_DEDUP_CDCX_AB3'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._AIRBYTE_UNIQUE_KEY = new_data_ids._AIRBYTE_UNIQUE_KEY + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro) + left join {{ ref('POS_DEDUP_CDCX_AB3') }} as inc_data on 1 = 0 + where _AIRBYTE_ACTIVE_ROW = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('POS_DEDUP_CDCX_AB3')) }} from new_data + union all + select {{ dbt_utils.star(ref('POS_DEDUP_CDCX_AB3')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('POS_DEDUP_CDCX_AB3') }} + -- POS_DEDUP_CDCX from {{ source('TEST_NORMALIZATION', '_AIRBYTE_RAW_POS_DEDUP_CDCX') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as {{ dbt_utils.type_string() }}), cast(_AB_CDC_UPDATED_AT as {{ dbt_utils.type_string() }}), cast(_AB_CDC_LOG_POS as {{ dbt_utils.type_string() }}) + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + {{ dbt_utils.surrogate_key([ + '_AIRBYTE_UNIQUE_KEY', + '_AIRBYTE_START_AT', + '_AIRBYTE_EMITTED_AT', '_AB_CDC_DELETED_AT', '_AB_CDC_UPDATED_AT', '_AB_CDC_LOG_POS' + ]) }} as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql similarity index 55% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql index 44185194f64f..1899392af6fc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql @@ -1,10 +1,14 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ 'ID', 'NAME', - adapter.quote('column`_\'with""_quotes'), '_AB_CDC_LSN', '_AB_CDC_UPDATED_AT', '_AB_CDC_DELETED_AT', @@ -12,4 +16,6 @@ select tmp.* from {{ ref('DEDUP_CDC_EXCLUDED_AB2') }} tmp -- DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql index 38db984c8b69..a78412000006 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -14,4 +19,6 @@ select tmp.* from {{ ref('DEDUP_EXCHANGE_RATE_AB2') }} tmp -- DEDUP_EXCHANGE_RATE +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql similarity index 61% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql index 743b21efdd28..c886fcf1eadf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql @@ -1,4 +1,9 @@ -{{ config(schema="_AIRBYTE_TEST_NORMALIZATION", tags=["top-level-intermediate"]) }} +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} -- SQL model to build a hash column based on the values of this record select {{ dbt_utils.surrogate_key([ @@ -12,4 +17,5 @@ select tmp.* from {{ ref('POS_DEDUP_CDCX_AB2') }} tmp -- POS_DEDUP_CDCX +where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql new file mode 100644 index 000000000000..d3b3c50580d0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql @@ -0,0 +1,17 @@ +{{ config( + cluster_by = ["_AIRBYTE_EMITTED_AT"], + unique_key = env_var('AIRBYTE_DEFAULT_UNIQUE_KEY', '_AIRBYTE_AB_ID'), + schema = "_AIRBYTE_TEST_NORMALIZATION", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'ID', + ]) }} as _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID, + tmp.* +from {{ ref('RENAMED_DEDUP_CDC_EXCLUDED_AB2') }} tmp +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 +{{ incremental_clause('_AIRBYTE_EMITTED_AT') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml index a8123e03f06b..c09293ca6726 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml @@ -10,3 +10,4 @@ sources: - name: _AIRBYTE_RAW_DEDUP_EXCHANGE_RATE - name: _AIRBYTE_RAW_EXCHANGE_RATE - name: _AIRBYTE_RAW_POS_DEDUP_CDCX + - name: _AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql new file mode 100644 index 000000000000..8d5282887fb0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB1.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_CDC_EXCLUDED as table_alias +-- DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql new file mode 100644 index 000000000000..57a50d43e5bc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB2.sql @@ -0,0 +1,28 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(NAME as + varchar +) as NAME, + cast(_AB_CDC_LSN as + float +) as _AB_CDC_LSN, + cast(_AB_CDC_UPDATED_AT as + float +) as _AB_CDC_UPDATED_AT, + cast(_AB_CDC_DELETED_AT as + float +) as _AB_CDC_DELETED_AT, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB1" +-- DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql new file mode 100644 index 000000000000..ee207b768a20 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB1.sql @@ -0,0 +1,21 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, + to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", + to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, + to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, + to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_DEDUP_EXCHANGE_RATE as table_alias +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql new file mode 100644 index 000000000000..66b09ec2508e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB2.sql @@ -0,0 +1,43 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(CURRENCY as + varchar +) as CURRENCY, + cast(nullif(DATE, '') as + date +) as DATE, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + when TIMESTAMP_COL = '' then NULL + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(HKD_SPECIAL___CHARACTERS as + varchar +) as HKD_SPECIAL___CHARACTERS, + cast(NZD as + float +) as NZD, + cast(USD as + float +) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB1" +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql new file mode 100644 index 000000000000..33e07a844a1c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB1.sql @@ -0,0 +1,21 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"currency"')) as CURRENCY, + to_varchar(get_path(parse_json(_airbyte_data), '"date"')) as DATE, + to_varchar(get_path(parse_json(_airbyte_data), '"timestamp_col"')) as TIMESTAMP_COL, + to_varchar(get_path(parse_json(_airbyte_data), '"HKD@spéçiäl & characters"')) as "HKD@spéçiäl & characters", + to_varchar(get_path(parse_json(_airbyte_data), '"HKD_special___characters"')) as HKD_SPECIAL___CHARACTERS, + to_varchar(get_path(parse_json(_airbyte_data), '"NZD"')) as NZD, + to_varchar(get_path(parse_json(_airbyte_data), '"USD"')) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_EXCHANGE_RATE as table_alias +-- EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql new file mode 100644 index 000000000000..f5ef6e473984 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB2.sql @@ -0,0 +1,43 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(CURRENCY as + varchar +) as CURRENCY, + cast(nullif(DATE, '') as + date +) as DATE, + case + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SSTZH') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{4}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + when TIMESTAMP_COL regexp '\\d{4}-\\d{2}-\\d{2}T(\\d{2}:){2}\\d{2}\\.\\d{1,7}(\\+|-)\\d{2}' then to_timestamp_tz(TIMESTAMP_COL, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + when TIMESTAMP_COL = '' then NULL + else to_timestamp_tz(TIMESTAMP_COL) + end as TIMESTAMP_COL + , + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(HKD_SPECIAL___CHARACTERS as + varchar +) as HKD_SPECIAL___CHARACTERS, + cast(NZD as + float +) as NZD, + cast(USD as + float +) as USD, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB1" +-- EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql new file mode 100644 index 000000000000..c1f0840c3677 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/EXCHANGE_RATE_AB3.sql @@ -0,0 +1,30 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(TIMESTAMP_COL as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as + varchar +), '') || '-' || coalesce(cast(NZD as + varchar +), '') || '-' || coalesce(cast(USD as + varchar +), '') as + varchar +)) as _AIRBYTE_EXCHANGE_RATE_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."EXCHANGE_RATE_AB2" tmp +-- EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql new file mode 100644 index 000000000000..93d03ade5063 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB1.sql @@ -0,0 +1,18 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + to_varchar(get_path(parse_json(_airbyte_data), '"name"')) as NAME, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_lsn"')) as _AB_CDC_LSN, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_updated_at"')) as _AB_CDC_UPDATED_AT, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_deleted_at"')) as _AB_CDC_DELETED_AT, + to_varchar(get_path(parse_json(_airbyte_data), '"_ab_cdc_log_pos"')) as _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX as table_alias +-- POS_DEDUP_CDCX +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql new file mode 100644 index 000000000000..82b6ca1c2619 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB2.sql @@ -0,0 +1,30 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + cast(NAME as + varchar +) as NAME, + cast(_AB_CDC_LSN as + float +) as _AB_CDC_LSN, + cast(_AB_CDC_UPDATED_AT as + float +) as _AB_CDC_UPDATED_AT, + cast(_AB_CDC_DELETED_AT as + float +) as _AB_CDC_DELETED_AT, + cast(_AB_CDC_LOG_POS as + float +) as _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB1" +-- POS_DEDUP_CDCX +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql new file mode 100644 index 000000000000..123971c4e5dc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB1.sql @@ -0,0 +1,14 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + to_varchar(get_path(parse_json(_airbyte_data), '"id"')) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED as table_alias +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql new file mode 100644 index 000000000000..35438f37c247 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_ctes/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB2.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(ID as + bigint +) as ID, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB1" +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..522240329e42 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","ID" = DBT_INTERNAL_SOURCE."ID","NAME" = DBT_INTERNAL_SOURCE."NAME","_AB_CDC_LSN" = DBT_INTERNAL_SOURCE."_AB_CDC_LSN","_AB_CDC_UPDATED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_UPDATED_AT","_AB_CDC_DELETED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_DELETED_AT","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql new file mode 100644 index 000000000000..00f8af4d282f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","ID" = DBT_INTERNAL_SOURCE."ID","CURRENCY" = DBT_INTERNAL_SOURCE."CURRENCY","DATE" = DBT_INTERNAL_SOURCE."DATE","TIMESTAMP_COL" = DBT_INTERNAL_SOURCE."TIMESTAMP_COL","HKD@spéçiäl & characters" = DBT_INTERNAL_SOURCE."HKD@spéçiäl & characters","HKD_SPECIAL___CHARACTERS" = DBT_INTERNAL_SOURCE."HKD_SPECIAL___CHARACTERS","NZD" = DBT_INTERNAL_SOURCE."NZD","USD" = DBT_INTERNAL_SOURCE."USD","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql new file mode 100644 index 000000000000..fc67dd05b1bf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."EXCHANGE_RATE__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_AB_ID = DBT_INTERNAL_DEST._AIRBYTE_AB_ID + + + + when matched then update set + "ID" = DBT_INTERNAL_SOURCE."ID","CURRENCY" = DBT_INTERNAL_SOURCE."CURRENCY","DATE" = DBT_INTERNAL_SOURCE."DATE","TIMESTAMP_COL" = DBT_INTERNAL_SOURCE."TIMESTAMP_COL","HKD@spéçiäl & characters" = DBT_INTERNAL_SOURCE."HKD@spéçiäl & characters","HKD_SPECIAL___CHARACTERS" = DBT_INTERNAL_SOURCE."HKD_SPECIAL___CHARACTERS","NZD" = DBT_INTERNAL_SOURCE."NZD","USD" = DBT_INTERNAL_SOURCE."USD","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_EXCHANGE_RATE_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_EXCHANGE_RATE_HASHID" + + + when not matched then insert + ("ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_EXCHANGE_RATE_HASHID") + values + ("ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_EXCHANGE_RATE_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql new file mode 100644 index 000000000000..3ddd6bd2a029 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","ID" = DBT_INTERNAL_SOURCE."ID","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "ID", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "ID", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..823c37218a10 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_SCD__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY_SCD = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY_SCD + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","_AIRBYTE_UNIQUE_KEY_SCD" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY_SCD","ID" = DBT_INTERNAL_SOURCE."ID","NAME" = DBT_INTERNAL_SOURCE."NAME","_AB_CDC_LSN" = DBT_INTERNAL_SOURCE."_AB_CDC_LSN","_AB_CDC_UPDATED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_UPDATED_AT","_AB_CDC_DELETED_AT" = DBT_INTERNAL_SOURCE."_AB_CDC_DELETED_AT","_AIRBYTE_START_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_START_AT","_AIRBYTE_END_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_END_AT","_AIRBYTE_ACTIVE_ROW" = DBT_INTERNAL_SOURCE."_AIRBYTE_ACTIVE_ROW","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "NAME", "_AB_CDC_LSN", "_AB_CDC_UPDATED_AT", "_AB_CDC_DELETED_AT", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql new file mode 100644 index 000000000000..f4b7a7831cdf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_SCD__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY_SCD = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY_SCD + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","_AIRBYTE_UNIQUE_KEY_SCD" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY_SCD","ID" = DBT_INTERNAL_SOURCE."ID","CURRENCY" = DBT_INTERNAL_SOURCE."CURRENCY","DATE" = DBT_INTERNAL_SOURCE."DATE","TIMESTAMP_COL" = DBT_INTERNAL_SOURCE."TIMESTAMP_COL","HKD@spéçiäl & characters" = DBT_INTERNAL_SOURCE."HKD@spéçiäl & characters","HKD_SPECIAL___CHARACTERS" = DBT_INTERNAL_SOURCE."HKD_SPECIAL___CHARACTERS","NZD" = DBT_INTERNAL_SOURCE."NZD","USD" = DBT_INTERNAL_SOURCE."USD","_AIRBYTE_START_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_START_AT","_AIRBYTE_END_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_END_AT","_AIRBYTE_ACTIVE_ROW" = DBT_INTERNAL_SOURCE."_AIRBYTE_ACTIVE_ROW","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "CURRENCY", "DATE", "TIMESTAMP_COL", "HKD@spéçiäl & characters", "HKD_SPECIAL___CHARACTERS", "NZD", "USD", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql new file mode 100644 index 000000000000..f95adde306f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_incremental/scd/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_SCD.sql @@ -0,0 +1,26 @@ +begin; + + + + + + + + merge into "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD" as DBT_INTERNAL_DEST + using "AIRBYTE_DATABASE".TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_SCD__dbt_tmp" as DBT_INTERNAL_SOURCE + on + DBT_INTERNAL_SOURCE._AIRBYTE_UNIQUE_KEY_SCD = DBT_INTERNAL_DEST._AIRBYTE_UNIQUE_KEY_SCD + + + + when matched then update set + "_AIRBYTE_UNIQUE_KEY" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY","_AIRBYTE_UNIQUE_KEY_SCD" = DBT_INTERNAL_SOURCE."_AIRBYTE_UNIQUE_KEY_SCD","ID" = DBT_INTERNAL_SOURCE."ID","_AIRBYTE_START_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_START_AT","_AIRBYTE_END_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_END_AT","_AIRBYTE_ACTIVE_ROW" = DBT_INTERNAL_SOURCE."_AIRBYTE_ACTIVE_ROW","_AIRBYTE_AB_ID" = DBT_INTERNAL_SOURCE."_AIRBYTE_AB_ID","_AIRBYTE_EMITTED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_EMITTED_AT","_AIRBYTE_NORMALIZED_AT" = DBT_INTERNAL_SOURCE."_AIRBYTE_NORMALIZED_AT","_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" = DBT_INTERNAL_SOURCE."_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID" + + + when not matched then insert + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") + values + ("_AIRBYTE_UNIQUE_KEY", "_AIRBYTE_UNIQUE_KEY_SCD", "ID", "_AIRBYTE_START_AT", "_AIRBYTE_END_AT", "_AIRBYTE_ACTIVE_ROW", "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", "_AIRBYTE_NORMALIZED_AT", "_AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID") + +; + commit; \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql new file mode 100644 index 000000000000..198a2627ab68 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/POS_DEDUP_CDCX.sql @@ -0,0 +1,25 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" as + (select * from( + +-- Final base SQL model +select + _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID +from "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" +-- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX +where 1 = 1 +and _AIRBYTE_ACTIVE_ROW = 1 + ) order by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX" cluster by (_AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql new file mode 100644 index 000000000000..24b58ed5bf4d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/scd/TEST_NORMALIZATION/POS_DEDUP_CDCX_SCD.sql @@ -0,0 +1,99 @@ + + + create or replace transient table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" as + (select * from( + +with + +input_data as ( + select * + from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" + -- POS_DEDUP_CDCX from "AIRBYTE_DATABASE".TEST_NORMALIZATION._AIRBYTE_RAW_POS_DEDUP_CDCX +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_EMITTED_AT as _AIRBYTE_START_AT, + lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) as _AIRBYTE_END_AT, + case when lag(_AIRBYTE_EMITTED_AT) over ( + partition by ID + order by + _AIRBYTE_EMITTED_AT is null asc, + _AIRBYTE_EMITTED_AT desc, + _AIRBYTE_EMITTED_AT desc, _AB_CDC_UPDATED_AT desc, _AB_CDC_LOG_POS desc + ) is null and _AB_CDC_DELETED_AT is null then 1 else 0 end as _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by _AIRBYTE_UNIQUE_KEY, _AIRBYTE_START_AT, _AIRBYTE_EMITTED_AT, cast(_AB_CDC_DELETED_AT as + varchar +), cast(_AB_CDC_UPDATED_AT as + varchar +), cast(_AB_CDC_LOG_POS as + varchar +) + order by _AIRBYTE_AB_ID + ) as _AIRBYTE_ROW_NUM, + md5(cast(coalesce(cast(_AIRBYTE_UNIQUE_KEY as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_START_AT as + varchar +), '') || '-' || coalesce(cast(_AIRBYTE_EMITTED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as + varchar +), '') as + varchar +)) as _AIRBYTE_UNIQUE_KEY_SCD, + scd_data.* + from scd_data +) +select + _AIRBYTE_UNIQUE_KEY, + _AIRBYTE_UNIQUE_KEY_SCD, + ID, + NAME, + _AB_CDC_LSN, + _AB_CDC_UPDATED_AT, + _AB_CDC_DELETED_AT, + _AB_CDC_LOG_POS, + _AIRBYTE_START_AT, + _AIRBYTE_END_AT, + _AIRBYTE_ACTIVE_ROW, + _AIRBYTE_AB_ID, + _AIRBYTE_EMITTED_AT, + convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, + _AIRBYTE_POS_DEDUP_CDCX_HASHID +from dedup_data where _AIRBYTE_ROW_NUM = 1 + ) order by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT) + ); + alter table "AIRBYTE_DATABASE".TEST_NORMALIZATION."POS_DEDUP_CDCX_SCD" cluster by (_AIRBYTE_ACTIVE_ROW, _AIRBYTE_UNIQUE_KEY, _AIRBYTE_EMITTED_AT); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql new file mode 100644 index 000000000000..522991af25cd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_CDC_EXCLUDED_AB3.sql @@ -0,0 +1,24 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(NAME as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LSN as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') as + varchar +)) as _AIRBYTE_DEDUP_CDC_EXCLUDED_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_CDC_EXCLUDED_AB2" tmp +-- DEDUP_CDC_EXCLUDED +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql new file mode 100644 index 000000000000..0b361061abf3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_AB3.sql @@ -0,0 +1,30 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(CURRENCY as + varchar +), '') || '-' || coalesce(cast(DATE as + varchar +), '') || '-' || coalesce(cast(TIMESTAMP_COL as + varchar +), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as + varchar +), '') || '-' || coalesce(cast(HKD_SPECIAL___CHARACTERS as + varchar +), '') || '-' || coalesce(cast(NZD as + varchar +), '') || '-' || coalesce(cast(USD as + varchar +), '') as + varchar +)) as _AIRBYTE_DEDUP_EXCHANGE_RATE_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."DEDUP_EXCHANGE_RATE_AB2" tmp +-- DEDUP_EXCHANGE_RATE +where 1 = 1 + + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql new file mode 100644 index 000000000000..c1189b3c1dfd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/POS_DEDUP_CDCX_AB3.sql @@ -0,0 +1,25 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') || '-' || coalesce(cast(NAME as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LSN as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_UPDATED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_DELETED_AT as + varchar +), '') || '-' || coalesce(cast(_AB_CDC_LOG_POS as + varchar +), '') as + varchar +)) as _AIRBYTE_POS_DEDUP_CDCX_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."POS_DEDUP_CDCX_AB2" tmp +-- POS_DEDUP_CDCX +where 1 = 1 + ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql new file mode 100644 index 000000000000..624b8d286061 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_views/TEST_NORMALIZATION/RENAMED_DEDUP_CDC_EXCLUDED_AB3.sql @@ -0,0 +1,16 @@ + + create or replace view "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB3" as ( + +-- SQL model to build a hash column based on the values of this record +select + md5(cast(coalesce(cast(ID as + varchar +), '') as + varchar +)) as _AIRBYTE_RENAMED_DEDUP_CDC_EXCLUDED_HASHID, + tmp.* +from "AIRBYTE_DATABASE"._AIRBYTE_TEST_NORMALIZATION."RENAMED_DEDUP_CDC_EXCLUDED_AB2" tmp +-- RENAMED_DEDUP_CDC_EXCLUDED +where 1 = 1 + + );