From 6ac01c03173ad95cfae25bdb960cdb41e2bf6100 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 28 Mar 2022 15:36:48 +0200 Subject: [PATCH 01/11] Upgrade MySQL to dbt 1.0.0 --- .../integration_tests/dbt_integration_test.py | 2 +- .../bases/base-normalization/mysql.Dockerfile | 9 ++------- .../normalization/NormalizationRunnerFactory.java | 2 +- docs/understanding-airbyte/basic-normalization.md | 1 + 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py index 295cc51a42d5..035e3dafa3df 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py @@ -401,7 +401,7 @@ def run_check_dbt_command(normalization_image: str, command: str, cwd: str, forc """ Run dbt subprocess while checking and counting for "ERROR", "FAIL" or "WARNING" printed in its outputs """ - if normalization_image.startswith("airbyte/normalization-oracle") or normalization_image.startswith("airbyte/normalization-mysql"): + if normalization_image.startswith("airbyte/normalization-oracle"): dbtAdditionalArgs = [] else: dbtAdditionalArgs = ["--event-buffer-size=10000"] diff --git a/airbyte-integrations/bases/base-normalization/mysql.Dockerfile b/airbyte-integrations/bases/base-normalization/mysql.Dockerfile index a5349f3e323c..8e0f4ebc5b11 100644 --- a/airbyte-integrations/bases/base-normalization/mysql.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/mysql.Dockerfile @@ -1,8 +1,4 @@ -# As of today, dbt-mysql doesn't support 1.0.0 -# IF YOU UPGRADE DBT, make sure to also edit these files: -# 1. Remove the "normalization-mysql" entry here https://github.com/airbytehq/airbyte/pull/11267/files#diff-9a3bcae8cb5c56aa30c00548e06eade6ad771f3d4f098f6867ae9a183049dfd8R404 -# 2. Check if oracle.Dockerfile is on DBT 1.0.0 yet; if it is, then revert this entire edit https://github.com/airbytehq/airbyte/pull/11267/files#diff-8880e85b2b5690accc6f15f9292a8589a6eb83564803d57c4ee74e2ee8ede09eR117-R130 -FROM fishtownanalytics/dbt:0.19.0 +FROM fishtownanalytics/dbt:1.0.0 COPY --from=airbyte/base-airbyte-protocol-python:0.1.1 /airbyte /airbyte # Install SSH Tunneling dependencies @@ -24,8 +20,7 @@ RUN pip install . WORKDIR /airbyte/normalization_code RUN pip install . -# Based of https://github.com/dbeatty10/dbt-mysql/tree/dev/0.19.0 -RUN pip install dbt-mysql==0.19.0 +RUN pip install dbt-mysql==1.0.0 WORKDIR /airbyte/normalization_code/dbt-template/ # Download external dbt dependencies diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index 04c5502dbb8e..069c0f9297aa 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.1.73"; + public static final String NORMALIZATION_VERSION = "0.1.74"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder() diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index f867af762d54..e681e60806d2 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -350,6 +350,7 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | |:----------------| :--- | :--- | :--- | :--- | +| 0.35.60-alpha | 0.1.74 | 2022-03-28 | [\#TBD](https://github.com/airbytehq/airbyte/pull/TBD) | Upgrade MySQL to dbt 1.0.0 | | 0.35.60-alpha | 0.1.73 | 2022-03-25 | [\#11267](https://github.com/airbytehq/airbyte/pull/11267) | Set `--event-buffer-size` to reduce memory usage | | 0.35.59-alpha | 0.1.72 | 2022-03-24 | [\#11093](https://github.com/airbytehq/airbyte/pull/11093) | Added Snowflake OAuth2.0 support | | 0.35.53-alpha | 0.1.71 | 2022-03-14 | [\#11077](https://github.com/airbytehq/airbyte/pull/11077) | Enable BigQuery to handle project ID embedded inside dataset ID | From 111580c919b7b9d6f532a6627396dda75c928cb9 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 28 Mar 2022 15:39:52 +0200 Subject: [PATCH 02/11] update changelog --- docs/understanding-airbyte/basic-normalization.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index e681e60806d2..c421730f87e0 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -350,7 +350,7 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | |:----------------| :--- | :--- | :--- | :--- | -| 0.35.60-alpha | 0.1.74 | 2022-03-28 | [\#TBD](https://github.com/airbytehq/airbyte/pull/TBD) | Upgrade MySQL to dbt 1.0.0 | +| 0.35.60-alpha | 0.1.74 | 2022-03-28 | [\#11470](https://github.com/airbytehq/airbyte/pull/11470) | Upgrade MySQL to dbt 1.0.0 | | 0.35.60-alpha | 0.1.73 | 2022-03-25 | [\#11267](https://github.com/airbytehq/airbyte/pull/11267) | Set `--event-buffer-size` to reduce memory usage | | 0.35.59-alpha | 0.1.72 | 2022-03-24 | [\#11093](https://github.com/airbytehq/airbyte/pull/11093) | Added Snowflake OAuth2.0 support | | 0.35.53-alpha | 0.1.71 | 2022-03-14 | [\#11077](https://github.com/airbytehq/airbyte/pull/11077) | Enable BigQuery to handle project ID embedded inside dataset ID | From ca38ffa2555a7eb56e5721709d6eed2291216fc8 Mon Sep 17 00:00:00 2001 From: alafanechere Date: Tue, 29 Mar 2022 23:41:37 +0200 Subject: [PATCH 03/11] update dbt_project.yml and packages.yml --- .../dbt_project.yml | 36 +++++++++---------- .../dbt-project-template-mysql/packages.yml | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml index b03cee8fe930..1539ed4f73c4 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml @@ -2,42 +2,42 @@ # the content will be overwritten by the transform function # Name your package! Package names should contain only lowercase characters -# and underscores. A good package name should reflect your organization's +# and underscores. A good package name should reflect your organization"s # name or the intended use of these models -name: 'airbyte_utils' -version: '1.0' +name: "airbyte_utils" +version: "1.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. Profiles contain # database connection information, and should be configured in the ~/.dbt/profiles.yml file -profile: 'normalize' +profile: "normalize" # These configurations specify where dbt should look for different types of files. -# The `source-paths` config, for example, states that source models can be found -# in the "models/" directory. You probably won't need to change these! -source-paths: ["models"] +# The `model-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won"t need to change these! +model-paths: ["models"] docs-paths: ["docs"] analysis-paths: ["analysis"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["data"] macro-paths: ["macros"] -target-path: "../build" # directory which will store compiled SQL files -log-path: "../logs" # directory which will store DBT logs -modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +packages-install-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies -clean-targets: # directories to be removed by `dbt clean` - - "build" - - "dbt_modules" +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" quoting: database: true -# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) -# all schemas should be unquoted + # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) + # all schemas should be unquoted schema: false identifier: true -# You can define configurations for models in the `source-paths` directory here. +# You can define configurations for models in the `model-paths` directory here. # Using these configurations, you can enable or disable models, change how they # are materialized, and more! models: @@ -60,4 +60,4 @@ models: +materialized: view vars: - dbt_utils_dispatch_list: ['airbyte_utils'] + dbt_utils_dispatch_list: ["airbyte_utils"] diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/packages.yml b/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/packages.yml index 13d4e69a45cb..33b4edd58c8c 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/packages.yml +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/packages.yml @@ -2,4 +2,4 @@ packages: - git: "https://github.com/fishtown-analytics/dbt-utils.git" - revision: 0.6.4 + revision: 0.8.2 From 3b1bc29478ef90446b62f6d958a09786f75da5aa Mon Sep 17 00:00:00 2001 From: alafanechere Date: Wed, 30 Mar 2022 13:26:16 +0200 Subject: [PATCH 04/11] fix entrypoint --- airbyte-integrations/bases/base-normalization/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-normalization/entrypoint.sh b/airbyte-integrations/bases/base-normalization/entrypoint.sh index 1784ab5f91df..5d72b036a906 100755 --- a/airbyte-integrations/bases/base-normalization/entrypoint.sh +++ b/airbyte-integrations/bases/base-normalization/entrypoint.sh @@ -114,6 +114,7 @@ function main() { openssh "${PROJECT_DIR}/ssh.json" trap 'closessh' EXIT + set +e # allow script to continue running even if next commands fail to run properly # We don't run dbt 1.0.x on all destinations (because their plugins don't support it yet) # So we need to only pass `--event-buffer-size` if it's supported by DBT. dbt --help | grep -E -- '--event-buffer-size' @@ -124,7 +125,6 @@ function main() { dbt_additional_args="" fi - set +e # allow script to continue running even if next commands fail to run properly # Run dbt to compile and execute the generated normalization models dbt ${dbt_additional_args} run --profiles-dir "${PROJECT_DIR}" --project-dir "${PROJECT_DIR}" DBT_EXIT_CODE=$? From 0c0cb5d18869e529d6f0168a05e3b6873f2fe7a7 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Thu, 7 Apr 2022 14:44:50 -0700 Subject: [PATCH 05/11] fix concat + regenerate output --- .../macros/cross_db_utils/concat.sql | 5 +++ .../mysql/test_nested_streams/dbt_project.yml | 36 +++++++++---------- ..._stream_with_co___long_names_partition.sql | 18 +++++----- ..._stream_with_co___names_partition_data.sql | 20 +++++------ ..._stream_with_co__ion_double_array_data.sql | 20 +++++------ ..._stream_with_co___long_names_partition.sql | 18 +++++----- ..._stream_with_co___names_partition_data.sql | 20 +++++------ ..._stream_with_co__ion_double_array_data.sql | 20 +++++------ .../mysql/test_simple_streams/dbt_project.yml | 36 +++++++++---------- .../test_normalization/exchange_rate.sql | 18 +++++----- .../dedup_exchange_rate_stg.sql | 14 ++++---- .../multiple_column_names_conflicts_stg.sql | 14 ++++---- .../test_normalization/exchange_rate.sql | 18 +++++----- .../dedup_exchange_rate_stg.sql | 14 ++++---- 14 files changed, 138 insertions(+), 133 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/concat.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/concat.sql index 536cebc7b23c..ebd5d265985c 100644 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/concat.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/concat.sql @@ -12,6 +12,11 @@ concat({{ fields|join(', ') }}) {%- endmacro %} +{% macro mysql__concat(fields) -%} + {#-- MySQL doesn't support the '||' operator as concatenation by default --#} + concat({{ fields|join(', ') }}) +{%- endmacro %} + {% macro sqlserver__concat(fields) -%} {#-- CONCAT() in SQL SERVER accepts from 2 to 254 arguments, we use batches for the main concat, to overcome the limit. --#} {% set concat_chunks = [] %} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml index b03cee8fe930..1539ed4f73c4 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml @@ -2,42 +2,42 @@ # the content will be overwritten by the transform function # Name your package! Package names should contain only lowercase characters -# and underscores. A good package name should reflect your organization's +# and underscores. A good package name should reflect your organization"s # name or the intended use of these models -name: 'airbyte_utils' -version: '1.0' +name: "airbyte_utils" +version: "1.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. Profiles contain # database connection information, and should be configured in the ~/.dbt/profiles.yml file -profile: 'normalize' +profile: "normalize" # These configurations specify where dbt should look for different types of files. -# The `source-paths` config, for example, states that source models can be found -# in the "models/" directory. You probably won't need to change these! -source-paths: ["models"] +# The `model-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won"t need to change these! +model-paths: ["models"] docs-paths: ["docs"] analysis-paths: ["analysis"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["data"] macro-paths: ["macros"] -target-path: "../build" # directory which will store compiled SQL files -log-path: "../logs" # directory which will store DBT logs -modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +packages-install-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies -clean-targets: # directories to be removed by `dbt clean` - - "build" - - "dbt_modules" +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" quoting: database: true -# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) -# all schemas should be unquoted + # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) + # all schemas should be unquoted schema: false identifier: true -# You can define configurations for models in the `source-paths` directory here. +# You can define configurations for models in the `model-paths` directory here. # Using these configurations, you can enable or disable models, change how they # are materialized, and more! models: @@ -60,4 +60,4 @@ models: +materialized: view vars: - dbt_utils_dispatch_list: ['airbyte_utils'] + dbt_utils_dispatch_list: ["airbyte_utils"] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql index 90323e9b56b1..9d4975c21dac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -4,7 +4,7 @@ test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` as ( -with __dbt__CTE__nested_stream_with_co_2g_names_partition_ab1 as ( +with __dbt__cte__nested_stream_with_co_2g_names_partition_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization.`nested_stream_with_co_1g_into_long_names_scd` @@ -24,10 +24,10 @@ from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` as table_ where 1 = 1 and `partition` is not null -), __dbt__CTE__nested_stream_with_co_2g_names_partition_ab2 as ( +), __dbt__cte__nested_stream_with_co_2g_names_partition_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__nested_stream_with_co_2g_names_partition_ab1 +-- depends_on: __dbt__cte__nested_stream_with_co_2g_names_partition_ab1 select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, @@ -37,23 +37,23 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab1 +from __dbt__cte__nested_stream_with_co_2g_names_partition_ab1 -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 -), __dbt__CTE__nested_stream_with_co_2g_names_partition_ab3 as ( +), __dbt__cte__nested_stream_with_co_2g_names_partition_ab3 as ( -- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__nested_stream_with_co_2g_names_partition_ab2 +-- depends_on: __dbt__cte__nested_stream_with_co_2g_names_partition_ab2 select md5(cast(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as char), ''), '-', coalesce(cast(double_array_data as char), ''), '-', coalesce(cast(`DATA` as char), '')) as char)) as _airbyte_partition_hashid, tmp.* -from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab2 tmp +from __dbt__cte__nested_stream_with_co_2g_names_partition_ab2 tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 )-- Final base SQL model --- depends_on: __dbt__CTE__nested_stream_with_co_2g_names_partition_ab3 +-- depends_on: __dbt__cte__nested_stream_with_co_2g_names_partition_ab3 select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, @@ -64,7 +64,7 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at, _airbyte_partition_hashid -from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab3 +from __dbt__cte__nested_stream_with_co_2g_names_partition_ab3 -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql index eea4c0c44827..b5bce12e294f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -4,7 +4,7 @@ test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` as ( -with __dbt__CTE__nested_stream_with_co_3es_partition_data_ab1 as ( +with __dbt__cte__nested_stream_with_co_3es_partition_data_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization.`nested_stream_with_co___long_names_partition` @@ -20,7 +20,7 @@ with numbers as ( select - p0.generated_number * pow(2, 0) + p0.generated_number * power(2, 0) + 1 @@ -68,10 +68,10 @@ left join joined on _airbyte_partition_hashid = joined._airbyte_hashid where 1 = 1 and `DATA` is not null -), __dbt__CTE__nested_stream_with_co_3es_partition_data_ab2 as ( +), __dbt__cte__nested_stream_with_co_3es_partition_data_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__nested_stream_with_co_3es_partition_data_ab1 +-- depends_on: __dbt__cte__nested_stream_with_co_3es_partition_data_ab1 select _airbyte_partition_hashid, cast(currency as char(1024)) as currency, @@ -80,23 +80,23 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab1 +from __dbt__cte__nested_stream_with_co_3es_partition_data_ab1 -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA where 1 = 1 -), __dbt__CTE__nested_stream_with_co_3es_partition_data_ab3 as ( +), __dbt__cte__nested_stream_with_co_3es_partition_data_ab3 as ( -- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__nested_stream_with_co_3es_partition_data_ab2 +-- depends_on: __dbt__cte__nested_stream_with_co_3es_partition_data_ab2 select md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_data_hashid, tmp.* -from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab2 tmp +from __dbt__cte__nested_stream_with_co_3es_partition_data_ab2 tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA where 1 = 1 )-- Final base SQL model --- depends_on: __dbt__CTE__nested_stream_with_co_3es_partition_data_ab3 +-- depends_on: __dbt__cte__nested_stream_with_co_3es_partition_data_ab3 select _airbyte_partition_hashid, currency, @@ -106,7 +106,7 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at, _airbyte_data_hashid -from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab3 +from __dbt__cte__nested_stream_with_co_3es_partition_data_ab3 -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql index 8819c2350a9a..279441127cad 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -4,7 +4,7 @@ test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` as ( -with __dbt__CTE__nested_stream_with_co_3double_array_data_ab1 as ( +with __dbt__cte__nested_stream_with_co_3double_array_data_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization.`nested_stream_with_co___long_names_partition` @@ -20,7 +20,7 @@ with numbers as ( select - p0.generated_number * pow(2, 0) + p0.generated_number * power(2, 0) + 1 @@ -68,10 +68,10 @@ left join joined on _airbyte_partition_hashid = joined._airbyte_hashid where 1 = 1 and double_array_data is not null -), __dbt__CTE__nested_stream_with_co_3double_array_data_ab2 as ( +), __dbt__cte__nested_stream_with_co_3double_array_data_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__nested_stream_with_co_3double_array_data_ab1 +-- depends_on: __dbt__cte__nested_stream_with_co_3double_array_data_ab1 select _airbyte_partition_hashid, cast(id as char(1024)) as id, @@ -80,23 +80,23 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__nested_stream_with_co_3double_array_data_ab1 +from __dbt__cte__nested_stream_with_co_3double_array_data_ab1 -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data where 1 = 1 -), __dbt__CTE__nested_stream_with_co_3double_array_data_ab3 as ( +), __dbt__cte__nested_stream_with_co_3double_array_data_ab3 as ( -- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__nested_stream_with_co_3double_array_data_ab2 +-- depends_on: __dbt__cte__nested_stream_with_co_3double_array_data_ab2 select md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(id as char), '')) as char)) as _airbyte_double_array_data_hashid, tmp.* -from __dbt__CTE__nested_stream_with_co_3double_array_data_ab2 tmp +from __dbt__cte__nested_stream_with_co_3double_array_data_ab2 tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data where 1 = 1 )-- Final base SQL model --- depends_on: __dbt__CTE__nested_stream_with_co_3double_array_data_ab3 +-- depends_on: __dbt__cte__nested_stream_with_co_3double_array_data_ab3 select _airbyte_partition_hashid, id, @@ -106,7 +106,7 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at, _airbyte_double_array_data_hashid -from __dbt__CTE__nested_stream_with_co_3double_array_data_ab3 +from __dbt__cte__nested_stream_with_co_3double_array_data_ab3 -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql index 90323e9b56b1..9d4975c21dac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -4,7 +4,7 @@ test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` as ( -with __dbt__CTE__nested_stream_with_co_2g_names_partition_ab1 as ( +with __dbt__cte__nested_stream_with_co_2g_names_partition_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization.`nested_stream_with_co_1g_into_long_names_scd` @@ -24,10 +24,10 @@ from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` as table_ where 1 = 1 and `partition` is not null -), __dbt__CTE__nested_stream_with_co_2g_names_partition_ab2 as ( +), __dbt__cte__nested_stream_with_co_2g_names_partition_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__nested_stream_with_co_2g_names_partition_ab1 +-- depends_on: __dbt__cte__nested_stream_with_co_2g_names_partition_ab1 select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, @@ -37,23 +37,23 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab1 +from __dbt__cte__nested_stream_with_co_2g_names_partition_ab1 -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 -), __dbt__CTE__nested_stream_with_co_2g_names_partition_ab3 as ( +), __dbt__cte__nested_stream_with_co_2g_names_partition_ab3 as ( -- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__nested_stream_with_co_2g_names_partition_ab2 +-- depends_on: __dbt__cte__nested_stream_with_co_2g_names_partition_ab2 select md5(cast(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as char), ''), '-', coalesce(cast(double_array_data as char), ''), '-', coalesce(cast(`DATA` as char), '')) as char)) as _airbyte_partition_hashid, tmp.* -from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab2 tmp +from __dbt__cte__nested_stream_with_co_2g_names_partition_ab2 tmp -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 )-- Final base SQL model --- depends_on: __dbt__CTE__nested_stream_with_co_2g_names_partition_ab3 +-- depends_on: __dbt__cte__nested_stream_with_co_2g_names_partition_ab3 select _airbyte_nested_strea__nto_long_names_hashid, double_array_data, @@ -64,7 +64,7 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at, _airbyte_partition_hashid -from __dbt__CTE__nested_stream_with_co_2g_names_partition_ab3 +from __dbt__cte__nested_stream_with_co_2g_names_partition_ab3 -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from test_normalization.`nested_stream_with_co_1g_into_long_names_scd` where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql index eea4c0c44827..b5bce12e294f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -4,7 +4,7 @@ test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` as ( -with __dbt__CTE__nested_stream_with_co_3es_partition_data_ab1 as ( +with __dbt__cte__nested_stream_with_co_3es_partition_data_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization.`nested_stream_with_co___long_names_partition` @@ -20,7 +20,7 @@ with numbers as ( select - p0.generated_number * pow(2, 0) + p0.generated_number * power(2, 0) + 1 @@ -68,10 +68,10 @@ left join joined on _airbyte_partition_hashid = joined._airbyte_hashid where 1 = 1 and `DATA` is not null -), __dbt__CTE__nested_stream_with_co_3es_partition_data_ab2 as ( +), __dbt__cte__nested_stream_with_co_3es_partition_data_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__nested_stream_with_co_3es_partition_data_ab1 +-- depends_on: __dbt__cte__nested_stream_with_co_3es_partition_data_ab1 select _airbyte_partition_hashid, cast(currency as char(1024)) as currency, @@ -80,23 +80,23 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab1 +from __dbt__cte__nested_stream_with_co_3es_partition_data_ab1 -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA where 1 = 1 -), __dbt__CTE__nested_stream_with_co_3es_partition_data_ab3 as ( +), __dbt__cte__nested_stream_with_co_3es_partition_data_ab3 as ( -- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__nested_stream_with_co_3es_partition_data_ab2 +-- depends_on: __dbt__cte__nested_stream_with_co_3es_partition_data_ab2 select md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(currency as char), '')) as char)) as _airbyte_data_hashid, tmp.* -from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab2 tmp +from __dbt__cte__nested_stream_with_co_3es_partition_data_ab2 tmp -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA where 1 = 1 )-- Final base SQL model --- depends_on: __dbt__CTE__nested_stream_with_co_3es_partition_data_ab3 +-- depends_on: __dbt__cte__nested_stream_with_co_3es_partition_data_ab3 select _airbyte_partition_hashid, currency, @@ -106,7 +106,7 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at, _airbyte_data_hashid -from __dbt__CTE__nested_stream_with_co_3es_partition_data_ab3 +from __dbt__cte__nested_stream_with_co_3es_partition_data_ab3 -- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from test_normalization.`nested_stream_with_co___long_names_partition` where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql index 8819c2350a9a..279441127cad 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -4,7 +4,7 @@ test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` as ( -with __dbt__CTE__nested_stream_with_co_3double_array_data_ab1 as ( +with __dbt__cte__nested_stream_with_co_3double_array_data_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization.`nested_stream_with_co___long_names_partition` @@ -20,7 +20,7 @@ with numbers as ( select - p0.generated_number * pow(2, 0) + p0.generated_number * power(2, 0) + 1 @@ -68,10 +68,10 @@ left join joined on _airbyte_partition_hashid = joined._airbyte_hashid where 1 = 1 and double_array_data is not null -), __dbt__CTE__nested_stream_with_co_3double_array_data_ab2 as ( +), __dbt__cte__nested_stream_with_co_3double_array_data_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__nested_stream_with_co_3double_array_data_ab1 +-- depends_on: __dbt__cte__nested_stream_with_co_3double_array_data_ab1 select _airbyte_partition_hashid, cast(id as char(1024)) as id, @@ -80,23 +80,23 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__nested_stream_with_co_3double_array_data_ab1 +from __dbt__cte__nested_stream_with_co_3double_array_data_ab1 -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data where 1 = 1 -), __dbt__CTE__nested_stream_with_co_3double_array_data_ab3 as ( +), __dbt__cte__nested_stream_with_co_3double_array_data_ab3 as ( -- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__nested_stream_with_co_3double_array_data_ab2 +-- depends_on: __dbt__cte__nested_stream_with_co_3double_array_data_ab2 select md5(cast(concat(coalesce(cast(_airbyte_partition_hashid as char), ''), '-', coalesce(cast(id as char), '')) as char)) as _airbyte_double_array_data_hashid, tmp.* -from __dbt__CTE__nested_stream_with_co_3double_array_data_ab2 tmp +from __dbt__cte__nested_stream_with_co_3double_array_data_ab2 tmp -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data where 1 = 1 )-- Final base SQL model --- depends_on: __dbt__CTE__nested_stream_with_co_3double_array_data_ab3 +-- depends_on: __dbt__cte__nested_stream_with_co_3double_array_data_ab3 select _airbyte_partition_hashid, id, @@ -106,7 +106,7 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at, _airbyte_double_array_data_hashid -from __dbt__CTE__nested_stream_with_co_3double_array_data_ab3 +from __dbt__cte__nested_stream_with_co_3double_array_data_ab3 -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from test_normalization.`nested_stream_with_co___long_names_partition` where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml index b03cee8fe930..1539ed4f73c4 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml @@ -2,42 +2,42 @@ # the content will be overwritten by the transform function # Name your package! Package names should contain only lowercase characters -# and underscores. A good package name should reflect your organization's +# and underscores. A good package name should reflect your organization"s # name or the intended use of these models -name: 'airbyte_utils' -version: '1.0' +name: "airbyte_utils" +version: "1.0" config-version: 2 # This setting configures which "profile" dbt uses for this project. Profiles contain # database connection information, and should be configured in the ~/.dbt/profiles.yml file -profile: 'normalize' +profile: "normalize" # These configurations specify where dbt should look for different types of files. -# The `source-paths` config, for example, states that source models can be found -# in the "models/" directory. You probably won't need to change these! -source-paths: ["models"] +# The `model-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won"t need to change these! +model-paths: ["models"] docs-paths: ["docs"] analysis-paths: ["analysis"] test-paths: ["tests"] -data-paths: ["data"] +seed-paths: ["data"] macro-paths: ["macros"] -target-path: "../build" # directory which will store compiled SQL files -log-path: "../logs" # directory which will store DBT logs -modules-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +packages-install-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies -clean-targets: # directories to be removed by `dbt clean` - - "build" - - "dbt_modules" +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" quoting: database: true -# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) -# all schemas should be unquoted + # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) + # all schemas should be unquoted schema: false identifier: true -# You can define configurations for models in the `source-paths` directory here. +# You can define configurations for models in the `model-paths` directory here. # Using these configurations, you can enable or disable models, change how they # are materialized, and more! models: @@ -60,4 +60,4 @@ models: +materialized: view vars: - dbt_utils_dispatch_list: ['airbyte_utils'] + dbt_utils_dispatch_list: ["airbyte_utils"] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index 0ee59f50d8e7..fd770070d75f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -4,7 +4,7 @@ test_normalization.`exchange_rate__dbt_tmp` as ( -with __dbt__CTE__exchange_rate_ab1 as ( +with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization._airbyte_raw_exchange_rate @@ -35,10 +35,10 @@ select from test_normalization._airbyte_raw_exchange_rate as table_alias -- exchange_rate where 1 = 1 -), __dbt__CTE__exchange_rate_ab2 as ( +), __dbt__cte__exchange_rate_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__exchange_rate_ab1 +-- depends_on: __dbt__cte__exchange_rate_ab1 select cast(id as signed @@ -65,21 +65,21 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__exchange_rate_ab1 +from __dbt__cte__exchange_rate_ab1 -- exchange_rate where 1 = 1 -), __dbt__CTE__exchange_rate_ab3 as ( +), __dbt__cte__exchange_rate_ab3 as ( -- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__exchange_rate_ab2 +-- depends_on: __dbt__cte__exchange_rate_ab2 select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), '')) as char)) as _airbyte_exchange_rate_hashid, tmp.* -from __dbt__CTE__exchange_rate_ab2 tmp +from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate where 1 = 1 )-- Final base SQL model --- depends_on: __dbt__CTE__exchange_rate_ab3 +-- depends_on: __dbt__cte__exchange_rate_ab3 select id, currency, @@ -96,7 +96,7 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at, _airbyte_exchange_rate_hashid -from __dbt__CTE__exchange_rate_ab3 +from __dbt__cte__exchange_rate_ab3 -- exchange_rate from test_normalization._airbyte_raw_exchange_rate where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 00d7e578d293..c736f9f1ec46 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,7 +1,7 @@ create view _airbyte_test_normalization.`dedup_exchange_rate_stg__dbt_tmp` as ( -with __dbt__CTE__dedup_exchange_rate_ab1 as ( +with __dbt__cte__dedup_exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization._airbyte_raw_dedup_exchange_rate @@ -31,10 +31,10 @@ from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate where 1 = 1 -), __dbt__CTE__dedup_exchange_rate_ab2 as ( +), __dbt__cte__dedup_exchange_rate_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__dedup_exchange_rate_ab1 +-- depends_on: __dbt__cte__dedup_exchange_rate_ab1 select cast(id as signed @@ -60,17 +60,17 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__dedup_exchange_rate_ab1 +from __dbt__cte__dedup_exchange_rate_ab1 -- dedup_exchange_rate where 1 = 1 )-- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__dedup_exchange_rate_ab2 +-- depends_on: __dbt__cte__dedup_exchange_rate_ab2 select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_dedup_exchange_rate_hashid, tmp.* -from __dbt__CTE__dedup_exchange_rate_ab2 tmp +from __dbt__cte__dedup_exchange_rate_ab2 tmp -- dedup_exchange_rate where 1 = 1 - ); + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql index 55de2948c5ef..b9356dd1f6ba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql @@ -1,7 +1,7 @@ create view _airbyte_test_normalization.`multiple_column_names_conflicts_stg__dbt_tmp` as ( -with __dbt__CTE__multiple_column_names_conflicts_ab1 as ( +with __dbt__cte__multiple_column_names_conflicts_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization._airbyte_raw_multiple_column_names_conflicts @@ -29,10 +29,10 @@ from test_normalization._airbyte_raw_multiple_column_names_conflicts as table_al -- multiple_column_names_conflicts where 1 = 1 -), __dbt__CTE__multiple_column_names_conflicts_ab2 as ( +), __dbt__cte__multiple_column_names_conflicts_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__multiple_column_names_conflicts_ab1 +-- depends_on: __dbt__cte__multiple_column_names_conflicts_ab1 select cast(id as signed @@ -56,17 +56,17 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__multiple_column_names_conflicts_ab1 +from __dbt__cte__multiple_column_names_conflicts_ab1 -- multiple_column_names_conflicts where 1 = 1 )-- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__multiple_column_names_conflicts_ab2 +-- depends_on: __dbt__cte__multiple_column_names_conflicts_ab2 select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(`User Id` as char), ''), '-', coalesce(cast(user_id as char), ''), '-', coalesce(cast(`User id_1` as char), ''), '-', coalesce(cast(`user id_2` as char), ''), '-', coalesce(cast(`User@Id` as char), ''), '-', coalesce(cast(userid as char), '')) as char)) as _airbyte_multiple_col__ames_conflicts_hashid, tmp.* -from __dbt__CTE__multiple_column_names_conflicts_ab2 tmp +from __dbt__cte__multiple_column_names_conflicts_ab2 tmp -- multiple_column_names_conflicts where 1 = 1 - ); + ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index 0ee59f50d8e7..fd770070d75f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -4,7 +4,7 @@ test_normalization.`exchange_rate__dbt_tmp` as ( -with __dbt__CTE__exchange_rate_ab1 as ( +with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization._airbyte_raw_exchange_rate @@ -35,10 +35,10 @@ select from test_normalization._airbyte_raw_exchange_rate as table_alias -- exchange_rate where 1 = 1 -), __dbt__CTE__exchange_rate_ab2 as ( +), __dbt__cte__exchange_rate_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__exchange_rate_ab1 +-- depends_on: __dbt__cte__exchange_rate_ab1 select cast(id as signed @@ -65,21 +65,21 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__exchange_rate_ab1 +from __dbt__cte__exchange_rate_ab1 -- exchange_rate where 1 = 1 -), __dbt__CTE__exchange_rate_ab3 as ( +), __dbt__cte__exchange_rate_ab3 as ( -- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__exchange_rate_ab2 +-- depends_on: __dbt__cte__exchange_rate_ab2 select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), '')) as char)) as _airbyte_exchange_rate_hashid, tmp.* -from __dbt__CTE__exchange_rate_ab2 tmp +from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate where 1 = 1 )-- Final base SQL model --- depends_on: __dbt__CTE__exchange_rate_ab3 +-- depends_on: __dbt__cte__exchange_rate_ab3 select id, currency, @@ -96,7 +96,7 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at, _airbyte_exchange_rate_hashid -from __dbt__CTE__exchange_rate_ab3 +from __dbt__cte__exchange_rate_ab3 -- exchange_rate from test_normalization._airbyte_raw_exchange_rate where 1 = 1 ) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 00d7e578d293..c736f9f1ec46 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,7 +1,7 @@ create view _airbyte_test_normalization.`dedup_exchange_rate_stg__dbt_tmp` as ( -with __dbt__CTE__dedup_exchange_rate_ab1 as ( +with __dbt__cte__dedup_exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization._airbyte_raw_dedup_exchange_rate @@ -31,10 +31,10 @@ from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate where 1 = 1 -), __dbt__CTE__dedup_exchange_rate_ab2 as ( +), __dbt__cte__dedup_exchange_rate_ab2 as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__CTE__dedup_exchange_rate_ab1 +-- depends_on: __dbt__cte__dedup_exchange_rate_ab1 select cast(id as signed @@ -60,17 +60,17 @@ select CURRENT_TIMESTAMP as _airbyte_normalized_at -from __dbt__CTE__dedup_exchange_rate_ab1 +from __dbt__cte__dedup_exchange_rate_ab1 -- dedup_exchange_rate where 1 = 1 )-- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__CTE__dedup_exchange_rate_ab2 +-- depends_on: __dbt__cte__dedup_exchange_rate_ab2 select md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), '')) as char)) as _airbyte_dedup_exchange_rate_hashid, tmp.* -from __dbt__CTE__dedup_exchange_rate_ab2 tmp +from __dbt__cte__dedup_exchange_rate_ab2 tmp -- dedup_exchange_rate where 1 = 1 - ); + ); \ No newline at end of file From 58ee603bf2b23bbf74c582506883c0a5770d3adf Mon Sep 17 00:00:00 2001 From: alafanechere Date: Mon, 11 Apr 2022 15:30:42 +0200 Subject: [PATCH 06/11] tmp max workers = 1 --- tools/bin/ci_integration_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/ci_integration_test.sh b/tools/bin/ci_integration_test.sh index 9d7d928b3e57..40b769473c61 100755 --- a/tools/bin/ci_integration_test.sh +++ b/tools/bin/ci_integration_test.sh @@ -19,7 +19,7 @@ else export SUB_BUILD="CONNECTORS_BASE" # avoid schema conflicts when multiple tests for normalization are run concurrently export RANDOM_TEST_SCHEMA="true" - ./gradlew --no-daemon --scan airbyteDocker + ./gradlew --max-workers=1 --no-daemon --scan airbyteDocker elif [[ "$connector" == *"bases"* ]]; then connector_name=$(echo $connector | cut -d / -f 2) selected_integration_test=$(echo "$all_integration_tests" | grep "^$connector_name$" || echo "") From 4402c20b4d9a984d4c3658fee844534995655370 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Tue, 12 Apr 2022 14:41:25 -0700 Subject: [PATCH 07/11] rebump --- airbyte-integrations/bases/base-normalization/Dockerfile | 2 +- .../workers/normalization/NormalizationRunnerFactory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index e7061f462af0..16a96353ef47 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.1.76 +LABEL io.airbyte.version=0.1.77 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index 37067525102e..0c1944eb0a79 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.1.76"; + public static final String NORMALIZATION_VERSION = "0.1.77"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder() From 6b5b34c0ee48e65e87bb82c45182bdc58b0ea560 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Fri, 22 Apr 2022 15:16:15 -0700 Subject: [PATCH 08/11] explicit airbyteDocker before runnig tests? --- tools/integrations/manage.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index fdb40bb8b3b2..7462ccd4cd9a 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -47,6 +47,11 @@ cmd_build() { echo "Skipping integration tests..." else echo "Running integration tests..." + + if test "$path" == "airbyte-integrations/bases/base-normalization"; then + ./gradlew --no-daemon --scan airbyteDocker + fi + ./gradlew --no-daemon "$(_to_gradle_path "$path" integrationTest)" fi } @@ -117,7 +122,7 @@ cmd_publish() { docker push "$versioned_image" docker push "$latest_image" fi - + # Checking if the image was successfully registered on DockerHub # see the description of this PR to understand why this is needed https://github.com/airbytehq/airbyte/pull/11654/ sleep 5 From 37cfd3936f6b52e1bb0e18d0245d5321e5b9d8cf Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Fri, 22 Apr 2022 15:53:44 -0700 Subject: [PATCH 09/11] only run normalization docker? --- tools/integrations/manage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index 7462ccd4cd9a..cf8656014022 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -49,7 +49,7 @@ cmd_build() { echo "Running integration tests..." if test "$path" == "airbyte-integrations/bases/base-normalization"; then - ./gradlew --no-daemon --scan airbyteDocker + ./gradlew --no-daemon --scan :airbyte-integrations:bases:base-normalization:airbyteDocker fi ./gradlew --no-daemon "$(_to_gradle_path "$path" integrationTest)" From 8a7c1d43201b239b09c20530de6d3f5ab103705c Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 15 Jun 2022 14:20:37 -0700 Subject: [PATCH 10/11] regenerate mysql output --- .../mysql/test_nested_streams/dbt_project.yml | 26 +++++++++---------- .../mysql/test_simple_streams/dbt_project.yml | 26 +++++++++---------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml index 937a423ec05d..88b2a40e2d11 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize -source-paths: - - models +model-paths: +- models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests -data-paths: - - data +- tests +seed-paths: +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs -modules-path: /dbt +packages-install-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: true schema: false @@ -42,7 +42,7 @@ models: +materialized: view vars: dbt_utils_dispatch_list: - - airbyte_utils + - airbyte_utils json_column: _airbyte_data models_to_source: nested_stream_with_co_1g_into_long_names_ab1: test_normalization._airbyte_raw_nested_s__lting_into_long_names diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml index bddbc9e03a80..5650b278a87c 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize -source-paths: - - models +model-paths: +- models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests -data-paths: - - data +- tests +seed-paths: +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs -modules-path: /dbt +packages-install-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: true schema: false @@ -42,7 +42,7 @@ models: +materialized: view vars: dbt_utils_dispatch_list: - - airbyte_utils + - airbyte_utils json_column: _airbyte_data models_to_source: exchange_rate_ab1: test_normalization._airbyte_raw_exchange_rate From d72be95bd5ebaed9eb4924c62bda934ceae6c7bb Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 15 Jun 2022 14:21:17 -0700 Subject: [PATCH 11/11] bump version --- airbyte-integrations/bases/base-normalization/Dockerfile | 2 +- .../bases/base-normalization/snowflake.Dockerfile | 2 +- .../workers/normalization/NormalizationRunnerFactory.java | 2 +- docs/understanding-airbyte/basic-normalization.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index cce30a21f7bb..9d4922f119ae 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile b/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile index 209ec40ffb19..bdc5a914889e 100644 --- a/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/snowflake.Dockerfile @@ -29,5 +29,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/normalization-snowflake diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index 27d6ff6cd8fd..d47dbd3633c4 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -14,7 +14,7 @@ public class NormalizationRunnerFactory { public static final String BASE_NORMALIZATION_IMAGE_NAME = "airbyte/normalization"; - public static final String NORMALIZATION_VERSION = "0.2.4"; + public static final String NORMALIZATION_VERSION = "0.2.5"; static final Map> NORMALIZATION_MAPPING = ImmutableMap.>builder() diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index 408d05f067a9..0d617c5330a0 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -353,7 +353,7 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | |:----------------| :--- | :--- | :--- | :--- | -| | 0.2.2 | 2022-05-19 | [\#11470](https://github.com/airbytehq/airbyte/pull/11470) | Upgrade MySQL to dbt 1.0.0 | +| | 0.2.5 | 2022-06-15 | [\#11470](https://github.com/airbytehq/airbyte/pull/11470) | Upgrade MySQL to dbt 1.0.0 | | | 0.2.4 | 2022-06-14 | [\#12846](https://github.com/airbytehq/airbyte/pull/12846) | CDC correctly deletes propagates deletions to final tables | | | 0.2.3 | 2022-06-10 | [\#11204](https://github.com/airbytehq/airbyte/pull/11204) | MySQL: add support for SSh tunneling | | | 0.2.2 | 2022-06-02 | [\#13289](https://github.com/airbytehq/airbyte/pull/13289) | BigQuery use `json_extract_string_array` for array of simple type elements |