diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index d550c4200498..c5130a35a808 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -4,9 +4,25 @@ USER root WORKDIR /tmp RUN apt-get update && apt-get install -y \ wget \ + curl \ unzip \ libaio-dev \ - libaio1 + libaio1 \ + gnupg \ + gnupg1 \ + gnupg2 + +# Install MS SQL Server dependencies +RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +RUN curl https://packages.microsoft.com/config/debian/10/prod.list > /etc/apt/sources.list.d/mssql-release.list +RUN apt-get update && ACCEPT_EULA=Y apt-get install -y \ + libgssapi-krb5-2 \ + unixodbc-dev \ + msodbcsql17 \ + mssql-tools +ENV PATH=$PATH:/opt/mssql-tools/bin + +# Install Oracle dependencies RUN mkdir -p /opt/oracle RUN wget https://download.oracle.com/otn_software/linux/instantclient/19600/instantclient-basic-linux.x64-19.6.0.0.0dbru.zip RUN unzip instantclient-basic-linux.x64-19.6.0.0.0dbru.zip -d /opt/oracle @@ -17,8 +33,8 @@ RUN pip install cx_Oracle COPY --from=airbyte/base-airbyte-protocol-python:0.1.1 /airbyte /airbyte +# Install SSH Tunneling dependencies RUN apt-get update && apt-get install -y jq sshpass - WORKDIR /airbyte COPY entrypoint.sh . COPY build/sshtunneling.sh . @@ -28,6 +44,7 @@ COPY normalization ./normalization COPY setup.py . COPY dbt-project-template/ ./dbt-template/ +# Install python dependencies WORKDIR /airbyte/base_python_structs RUN pip install . @@ -35,6 +52,7 @@ WORKDIR /airbyte/normalization_code RUN pip install . RUN pip install dbt-oracle==0.4.3 RUN pip install git+https://github.com/dbeatty10/dbt-mysql@96655ea9f7fca7be90c9112ce8ffbb5aac1d3716#egg=dbt-mysql +RUN pip install dbt-sqlserver==0.19.3 WORKDIR /airbyte/normalization_code/dbt-template/ @@ -45,5 +63,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.1.49 +LABEL io.airbyte.version=0.1.50 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-integrations/bases/base-normalization/README.md b/airbyte-integrations/bases/base-normalization/README.md index fe73edd79fd4..237c5166e7bc 100644 --- a/airbyte-integrations/bases/base-normalization/README.md +++ b/airbyte-integrations/bases/base-normalization/README.md @@ -9,6 +9,25 @@ Related documentation on normalization is available here: Below are short descriptions of the kind of tests that may be affected by changes to the normalization code. +### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python3 -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + ## Unit Tests Unit tests are automatically included when building the normalization project. @@ -56,6 +75,7 @@ allowed characters, if quotes are needed or not, and the length limitations: - [snowflake](../../../docs/integrations/destinations/snowflake.md) - [mysql](../../../docs/integrations/destinations/mysql.md) - [oracle](../../../docs/integrations/destinations/oracle.md) +- [mssql](../../../docs/integrations/destinations/mssql.md) Rules about truncations, for example for both of these strings which are too long for the postgres 64 limit: - `Aaaa_Bbbb_Cccc_Dddd_Eeee_Ffff_Gggg_Hhhh_Iiii` diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index df65bbeadd03..a6d1e38318c5 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -44,6 +44,7 @@ task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker' dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' + dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' } diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/array.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/array.sql index cbe1d7d1d92c..84148323f1f7 100644 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/array.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/array.sql @@ -4,6 +4,7 @@ - Snowflake: flatten() -> https://docs.snowflake.com/en/sql-reference/functions/flatten.html - Redshift: -> https://blog.getdbt.com/how-to-unnest-arrays-in-redshift/ - postgres: unnest() -> https://www.postgresqltutorial.com/postgresql-array/ + - MSSQL: openjson() –> https://docs.microsoft.com/en-us/sql/relational-databases/json/validate-query-and-change-json-data-with-built-in-functions-sql-server?view=sql-server-ver15 #} {# cross_join_unnest ------------------------------------------------- #} @@ -44,6 +45,17 @@ cross join table(flatten({{ array_col }})) as {{ array_col }} {%- endmacro %} +{% macro sqlserver__cross_join_unnest(stream_name, array_col) -%} +{# https://docs.microsoft.com/en-us/sql/relational-databases/json/convert-json-data-to-rows-and-columns-with-openjson-sql-server?view=sql-server-ver15#option-1---openjson-with-the-default-output #} + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON({{ array_col }}) + ) AS {{ array_col }} +{%- endmacro %} + {# unnested_column_value -- this macro is related to unnest_cte #} {% macro unnested_column_value(column_col) -%} @@ -74,6 +86,11 @@ {{ column_col }} {%- endmacro %} +{% macro sqlserver__unnested_column_value(column_col) -%} + {# unnested array/sub_array will be located in `value` column afterwards, we need to address to it #} + {{ column_col }}.value +{%- endmacro %} + {# unnest_cte ------------------------------------------------- #} {% macro unnest_cte(table_name, stream_name, column_col) -%} diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/concat.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/concat.sql index fc81a7e0491a..7f14a20cd16e 100644 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/concat.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/concat.sql @@ -14,3 +14,13 @@ {% macro postgres__concat(fields) %} {{ dbt_utils.alternative_concat(fields) }} {% endmacro %} + +{% macro sqlserver__concat(fields) -%} + {#-- CONCAT() in SQL SERVER accepts from 2 to 254 arguments, we use batches for the main concat, to overcome the limit. --#} + {% set concat_chunks = [] %} + {% for chunk in fields|batch(253) -%} + {% set _ = concat_chunks.append( "concat(" ~ chunk|join(', ') ~ ",'')" ) %} + {% endfor %} + + concat({{ concat_chunks|join(', ') }}, '') +{%- endmacro %} diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/datatypes.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/datatypes.sql index 3ae63c7ae5d9..28b734ee6fe2 100644 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/datatypes.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/datatypes.sql @@ -28,6 +28,10 @@ json {%- endmacro -%} +{%- macro sqlserver__type_json() -%} + VARCHAR(max) +{%- endmacro -%} + {# string ------------------------------------------------- #} @@ -39,6 +43,10 @@ varchar2(4000) {%- endmacro -%} +{% macro sqlserver__type_string() %} + VARCHAR(max) +{%- endmacro -%} + {# float ------------------------------------------------- #} {% macro mysql__type_float() %} @@ -69,17 +77,23 @@ {% endmacro %} -{# numeric ------------------------------------------------- #} +{# numeric ------------------------------------------------- --#} {% macro mysql__type_numeric() %} float {% endmacro %} -{# timestamp ------------------------------------------------- #} +{# timestamp ------------------------------------------------- --#} {% macro mysql__type_timestamp() %} time {% endmacro %} +{%- macro sqlserver__type_timestamp() -%} + {#-- in TSQL timestamp is really datetime --#} + {#-- https://docs.microsoft.com/en-us/sql/t-sql/functions/date-and-time-data-types-and-functions-transact-sql?view=sql-server-ver15#DateandTimeDataTypes --#} + datetime +{%- endmacro -%} + {# timestamp with time zone ------------------------------------------------- #} @@ -95,7 +109,7 @@ timestamp {% endmacro %} -{# MySQL doesnt allow cast operation to work with TIMESTAMP so we have to use char #} +{#-- MySQL doesnt allow cast operation to work with TIMESTAMP so we have to use char --#} {%- macro mysql__type_timestamp_with_timezone() -%} char {%- endmacro -%} @@ -104,6 +118,12 @@ varchar2(4000) {% endmacro %} +{%- macro sqlserver__type_timestamp_with_timezone() -%} + {#-- in TSQL timestamp is really datetime or datetime2 --#} + {#-- https://docs.microsoft.com/en-us/sql/t-sql/functions/date-and-time-data-types-and-functions-transact-sql?view=sql-server-ver15#DateandTimeDataTypes --#} + datetime +{%- endmacro -%} + {# date ------------------------------------------------- #} @@ -118,3 +138,7 @@ {% macro oracle__type_date() %} varchar2(4000) {% endmacro %} + +{%- macro sqlserver__type_date() -%} + date +{%- endmacro -%} diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/hash.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/hash.sql new file mode 100644 index 000000000000..184888794b9f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/hash.sql @@ -0,0 +1,5 @@ +{# converting hash in varchar _macro #} + +{% macro sqlserver__hash(field) -%} + convert(varchar(32), HashBytes('md5', coalesce(cast({{field}} as {{dbt_utils.type_string()}}), '')), 2) +{%- endmacro %} \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql index 74ce5968545d..e0d27c4ae3b3 100644 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/json_operations.sql @@ -57,6 +57,15 @@ {{ "'\"" ~ str_list|join('"."') ~ "\"'" }} {%- endmacro %} +{% macro sqlserver__format_json_path(json_path_list) -%} + {# -- '$."x"."y"."z"' #} + {%- set str_list = [] -%} + {%- for json_path in json_path_list -%} + {%- if str_list.append(json_path.replace("'", "''").replace('"', '\\"')) -%} {%- endif -%} + {%- endfor -%} + {{ "'$.\"" ~ str_list|join(".") ~ "\"'" }} +{%- endmacro %} + {# json_extract ------------------------------------------------- #} {% macro json_extract(from_table, json_column, json_path_list, normalized_json_path) -%} @@ -111,6 +120,10 @@ {% endif -%} {%- endmacro %} +{% macro sqlserver__json_extract(from_table, json_column, json_path_list, normalized_json_path) -%} + json_query({{ json_column }}, {{ format_json_path(json_path_list) }}) +{%- endmacro %} + {# json_extract_scalar ------------------------------------------------- #} {% macro json_extract_scalar(json_column, json_path_list, normalized_json_path) -%} @@ -145,6 +158,10 @@ to_varchar(get_path(parse_json({{ json_column }}), {{ format_json_path(json_path_list) }})) {%- endmacro %} +{% macro sqlserver__json_extract_scalar(json_column, json_path_list, normalized_json_path) -%} + json_value({{ json_column }}, {{ format_json_path(json_path_list) }}) +{%- endmacro %} + {# json_extract_array ------------------------------------------------- #} {% macro json_extract_array(json_column, json_path_list, normalized_json_path) -%} @@ -178,3 +195,7 @@ {% macro snowflake__json_extract_array(json_column, json_path_list, normalized_json_path) -%} get_path(parse_json({{ json_column }}), {{ format_json_path(json_path_list) }}) {%- endmacro %} + +{% macro sqlserver__json_extract_array(json_column, json_path_list, normalized_json_path) -%} + json_query({{ json_column }}, {{ format_json_path(json_path_list) }}) +{%- endmacro %} diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/type_conversions.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/type_conversions.sql index 3e7c572c0ddc..65df78c59a62 100644 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/type_conversions.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/type_conversions.sql @@ -29,6 +29,10 @@ cast({{ array_column }} as varchar2(4000)) {%- endmacro %} +{% macro sqlserver__array_to_string(array_column) -%} + cast({{ array_column }} as {{dbt_utils.type_string()}}) +{%- endmacro %} + {# cast_to_boolean ------------------------------------------------- #} {% macro cast_to_boolean(field) -%} {{ adapter.dispatch('cast_to_boolean')(field) }} @@ -47,3 +51,8 @@ {% macro redshift__cast_to_boolean(field) -%} cast(decode({{ field }}, 'true', '1', 'false', '0')::integer as boolean) {%- endmacro %} + +{# -- MS SQL Server does not support converting string directly to boolean, it must be casted as bit #} +{% macro sqlserver__cast_to_boolean(field) -%} + cast({{ field }} as bit) +{%- endmacro %} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py index b5d37a42cb78..00b02cc5e277 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py @@ -24,7 +24,7 @@ class DbtIntegrationTest(object): def __init__(self): self.target_schema = "test_normalization" self.container_prefix = f"test_normalization_db_{self.random_string(3)}" - self.db_names = ["postgres", "mysql"] + self.db_names = ["postgres", "mysql", "mssql"] @staticmethod def random_string(length: int) -> str: @@ -33,6 +33,7 @@ def random_string(length: int) -> str: def setup_db(self): self.setup_postgres_db() self.setup_mysql_db() + self.setup_mssql_db() def setup_postgres_db(self): print("Starting localhost postgres container for tests") @@ -68,7 +69,8 @@ def setup_postgres_db(self): ] print("Executing: ", " ".join(commands)) subprocess.call(commands) - time.sleep(5) + print("....Waiting for Postgres DB to start...15 sec") + time.sleep(15) if not os.path.exists("../secrets"): os.makedirs("../secrets") @@ -104,13 +106,79 @@ def setup_mysql_db(self): ] print("Executing: ", " ".join(commands)) subprocess.call(commands) - time.sleep(5) + print("....Waiting for MySQL DB to start...15 sec") + time.sleep(15) if not os.path.exists("../secrets"): os.makedirs("../secrets") with open("../secrets/mysql.json", "w") as fh: fh.write(json.dumps(config)) + def setup_mssql_db(self): + print("Starting localhost MS SQL Server container for tests") + port = self.find_free_port() + config = { + "host": "localhost", + "username": "SA", + "password": "MyStr0ngP@ssw0rd", + "port": port, + "database": self.target_schema, + "schema": self.target_schema, + } + + command_start_container = [ + "docker", + "run", + "--rm", + "--name", + f"{self.container_prefix}_mssql", + "-h", + f"{self.container_prefix}_mssql", + "-e", + "ACCEPT_EULA='Y'", + "-e", + f"SA_PASSWORD='{config['password']}'", + "-e", + "MSSQL_PID='Standard'", + "-p", + f"{config['port']}:1433", + "-d", + "mcr.microsoft.com/mssql/server:2019-GA-ubuntu-16.04", + ] + # Run additional commands to prepare the table + command_create_db = [ + "docker", + "exec", + f"{self.container_prefix}_mssql", + "/opt/mssql-tools/bin/sqlcmd", + "-S", + config["host"], + "-U", + config["username"], + "-P", + config["password"], + "-Q", + f"CREATE DATABASE [{config['database']}]", + ] + + # cmds & parameters + cmd_start_container = " ".join(command_start_container) + wait_sec = 30 + # run the docker container + print("Executing: ", cmd_start_container) + subprocess.check_call(cmd_start_container, shell=True) + # wait for service is available + print(f"....Waiting for MS SQL Server to start...{wait_sec} sec") + time.sleep(wait_sec) + # create test db + print("Executing: ", " ".join(command_create_db)) + subprocess.call(command_create_db) + + if not os.path.exists("../secrets"): + os.makedirs("../secrets") + with open("../secrets/mssql.json", "w") as fh: + fh.write(json.dumps(config)) + @staticmethod def find_free_port(): """ diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..c4aa05fcbac9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,90 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_array__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_array__dbt_tmp_temp_view" as + +with __dbt__CTE__conflict_stream_array_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."conflict_stream_array"'') as conflict_stream_array, + _airbyte_emitted_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array as table_alias +-- conflict_stream_array +), __dbt__CTE__conflict_stream_array_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + conflict_stream_array, + _airbyte_emitted_at +from __dbt__CTE__conflict_stream_array_ab1 +-- conflict_stream_array +), __dbt__CTE__conflict_stream_array_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(conflict_stream_array as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_array_hashid, + tmp.* +from __dbt__CTE__conflict_stream_array_ab2 tmp +-- conflict_stream_array +)-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_emitted_at, + _airbyte_conflict_stream_array_hashid +from __dbt__CTE__conflict_stream_array_ab3 +-- conflict_stream_array from "test_normalization".test_normalization._airbyte_raw_conflict_stream_array + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_array__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_array__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_array__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_array__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_array__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_array__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_array__dbt_tmp.test_normalization_conflict_stream_array__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_array__dbt_tmp_cci + ON test_normalization.conflict_stream_array__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..2b8a645d6bf8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,89 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name__dbt_tmp_temp_view" as + +with __dbt__CTE__conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."conflict_stream_name"'') as conflict_stream_name, + _airbyte_emitted_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name as table_alias +-- conflict_stream_name +), __dbt__CTE__conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, + _airbyte_emitted_at +from __dbt__CTE__conflict_stream_name_ab1 +-- conflict_stream_name +), __dbt__CTE__conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_hashid, + tmp.* +from __dbt__CTE__conflict_stream_name_ab2 tmp +-- conflict_stream_name +)-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_emitted_at, + _airbyte_conflict_stream_name_hashid +from __dbt__CTE__conflict_stream_name_ab3 +-- conflict_stream_name from "test_normalization".test_normalization._airbyte_raw_conflict_stream_name + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql new file mode 100644 index 000000000000..1d91b998cd6c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -0,0 +1,90 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" as + +with __dbt__CTE__conflict_stream_name____conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + json_value(conflict_stream_name, ''$."groups"'') as groups, + _airbyte_emitted_at +from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" as table_alias +where conflict_stream_name is not null +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +), __dbt__CTE__conflict_stream_name____conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as + VARCHAR(max)) as groups, + _airbyte_emitted_at +from __dbt__CTE__conflict_stream_name____conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +), __dbt__CTE__conflict_stream_name____conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_conflict_stream_name_2_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(groups as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from __dbt__CTE__conflict_stream_name____conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name +)-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_emitted_at, + _airbyte_conflict_stream_name_3_hashid +from __dbt__CTE__conflict_stream_name____conflict_stream_name_ab3 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name" + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name____conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name____conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name____conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..f5e08f3ae73a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,89 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" as + +with __dbt__CTE__conflict_stream_name_conflict_stream_name_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + json_query(conflict_stream_name, ''$."conflict_stream_name"'') as conflict_stream_name, + _airbyte_emitted_at +from "test_normalization".test_normalization."conflict_stream_name" as table_alias +where conflict_stream_name is not null +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +), __dbt__CTE__conflict_stream_name_conflict_stream_name_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as VARCHAR(max)) as conflict_stream_name, + _airbyte_emitted_at +from __dbt__CTE__conflict_stream_name_conflict_stream_name_ab1 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +), __dbt__CTE__conflict_stream_name_conflict_stream_name_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_conflict_stream_name_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_name as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from __dbt__CTE__conflict_stream_name_conflict_stream_name_ab2 tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name +)-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_emitted_at, + _airbyte_conflict_stream_name_2_hashid +from __dbt__CTE__conflict_stream_name_conflict_stream_name_ab3 +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from "test_normalization".test_normalization."conflict_stream_name" + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_name_conflict_stream_name__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp.test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_name_conflict_stream_name__dbt_tmp_cci + ON test_normalization.conflict_stream_name_conflict_stream_name__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..c2ccb6757762 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,91 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp"','U') is not null + begin + drop table test_normalization."conflict_stream_scalar__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" as + +with __dbt__CTE__conflict_stream_scalar_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."conflict_stream_scalar"'') as conflict_stream_scalar, + _airbyte_emitted_at +from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar as table_alias +-- conflict_stream_scalar +), __dbt__CTE__conflict_stream_scalar_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast(conflict_stream_scalar as + bigint +) as conflict_stream_scalar, + _airbyte_emitted_at +from __dbt__CTE__conflict_stream_scalar_ab1 +-- conflict_stream_scalar +), __dbt__CTE__conflict_stream_scalar_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(conflict_stream_scalar as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from __dbt__CTE__conflict_stream_scalar_ab2 tmp +-- conflict_stream_scalar +)-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_emitted_at, + _airbyte_conflict_stream_scalar_hashid +from __dbt__CTE__conflict_stream_scalar_ab3 +-- conflict_stream_scalar from "test_normalization".test_normalization._airbyte_raw_conflict_stream_scalar + '); + + SELECT * INTO "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp" FROM + "test_normalization".test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."conflict_stream_scalar__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."conflict_stream_scalar__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_conflict_stream_scalar__dbt_tmp_cci' + AND object_id=object_id('test_normalization_conflict_stream_scalar__dbt_tmp') + ) + DROP index test_normalization.conflict_stream_scalar__dbt_tmp.test_normalization_conflict_stream_scalar__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_conflict_stream_scalar__dbt_tmp_cci + ON test_normalization.conflict_stream_scalar__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..00e15bdbb3c5 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,59 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded__dbt_tmp"','U') is not null + begin + drop table test_normalization."dedup_cdc_excluded__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid +from "test_normalization".test_normalization."dedup_cdc_excluded_scd" +-- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded +where _airbyte_active_row = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded__dbt_tmp" FROM + "test_normalization".test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."dedup_cdc_excluded__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded__dbt_tmp_cci' + AND object_id=object_id('test_normalization_dedup_cdc_excluded__dbt_tmp') + ) + DROP index test_normalization.dedup_cdc_excluded__dbt_tmp.test_normalization_dedup_cdc_excluded__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded__dbt_tmp_cci + ON test_normalization.dedup_cdc_excluded__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..8b41eb8e7523 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,130 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_scd__dbt_tmp"','U') is not null + begin + drop table test_normalization."dedup_cdc_excluded_scd__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" as + +with __dbt__CTE__dedup_cdc_excluded_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."name"'') as name, + json_value(_airbyte_data, ''$."_ab_cdc_lsn"'') as _ab_cdc_lsn, + json_value(_airbyte_data, ''$."_ab_cdc_updated_at"'') as _ab_cdc_updated_at, + json_value(_airbyte_data, ''$."_ab_cdc_deleted_at"'') as _ab_cdc_deleted_at, + _airbyte_emitted_at +from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded as table_alias +-- dedup_cdc_excluded +), __dbt__CTE__dedup_cdc_excluded_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(name as + VARCHAR(max)) as name, + cast(_ab_cdc_lsn as + float +) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as + float +) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as + float +) as _ab_cdc_deleted_at, + _airbyte_emitted_at +from __dbt__CTE__dedup_cdc_excluded_ab1 +-- dedup_cdc_excluded +), __dbt__CTE__dedup_cdc_excluded_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(name as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_lsn as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_updated_at as + VARCHAR(max)), ''''), ''-'', coalesce(cast(_ab_cdc_deleted_at as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from __dbt__CTE__dedup_cdc_excluded_ab2 tmp +-- dedup_cdc_excluded +), __dbt__CTE__dedup_cdc_excluded_ab4 as ( + +-- SQL model to prepare for deduplicating records based on the hash record column +select + row_number() over ( + partition by _airbyte_dedup_cdc_excluded_hashid + order by _airbyte_emitted_at asc + ) as _airbyte_row_num, + tmp.* +from __dbt__CTE__dedup_cdc_excluded_ab3 tmp +-- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded +)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key +select + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid +from __dbt__CTE__dedup_cdc_excluded_ab4 +-- dedup_cdc_excluded from "test_normalization".test_normalization._airbyte_raw_dedup_cdc_excluded +where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."dedup_cdc_excluded_scd__dbt_tmp" FROM + "test_normalization".test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."dedup_cdc_excluded_scd__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_dedup_cdc_excluded_scd__dbt_tmp_cci' + AND object_id=object_id('test_normalization_dedup_cdc_excluded_scd__dbt_tmp') + ) + DROP index test_normalization.dedup_cdc_excluded_scd__dbt_tmp.test_normalization_dedup_cdc_excluded_scd__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_cdc_excluded_scd__dbt_tmp_cci + ON test_normalization.dedup_cdc_excluded_scd__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..8d67ed36355d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,62 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate__dbt_tmp"','U') is not null + begin + drop table test_normalization."dedup_exchange_rate__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid +from "test_normalization".test_normalization."dedup_exchange_rate_scd" +-- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate +where _airbyte_active_row = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."dedup_exchange_rate__dbt_tmp" FROM + "test_normalization".test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."dedup_exchange_rate__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_dedup_exchange_rate__dbt_tmp_cci' + AND object_id=object_id('test_normalization_dedup_exchange_rate__dbt_tmp') + ) + DROP index test_normalization.dedup_exchange_rate__dbt_tmp.test_normalization_dedup_exchange_rate__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_exchange_rate__dbt_tmp_cci + ON test_normalization.dedup_exchange_rate__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..38c4e21a95f0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,145 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_scd__dbt_tmp"','U') is not null + begin + drop table test_normalization."dedup_exchange_rate_scd__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" as + +with __dbt__CTE__dedup_exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + _airbyte_emitted_at +from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +-- dedup_exchange_rate +), __dbt__CTE__dedup_exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse("date" as date) as "date", + try_parse(timestamp_col as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_emitted_at +from __dbt__CTE__dedup_exchange_rate_ab1 +-- dedup_exchange_rate +), __dbt__CTE__dedup_exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as + VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from __dbt__CTE__dedup_exchange_rate_ab2 tmp +-- dedup_exchange_rate +), __dbt__CTE__dedup_exchange_rate_ab4 as ( + +-- SQL model to prepare for deduplicating records based on the hash record column +select + row_number() over ( + partition by _airbyte_dedup_exchange_rate_hashid + order by _airbyte_emitted_at asc + ) as _airbyte_row_num, + tmp.* +from __dbt__CTE__dedup_exchange_rate_ab3 tmp +-- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate +)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key +select + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + "date" as _airbyte_start_at, + lag("date") over ( + partition by id, currency, cast(nzd as + VARCHAR(max)) + order by "date" desc, "date" desc, _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag("date") over ( + partition by id, currency, cast(nzd as + VARCHAR(max)) + order by "date" desc, "date" desc, _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid +from __dbt__CTE__dedup_exchange_rate_ab4 +-- dedup_exchange_rate from "test_normalization".test_normalization._airbyte_raw_dedup_exchange_rate +where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."dedup_exchange_rate_scd__dbt_tmp" FROM + "test_normalization".test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."dedup_exchange_rate_scd__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_dedup_exchange_rate_scd__dbt_tmp_cci' + AND object_id=object_id('test_normalization_dedup_exchange_rate_scd__dbt_tmp') + ) + DROP index test_normalization.dedup_exchange_rate_scd__dbt_tmp.test_normalization_dedup_exchange_rate_scd__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_dedup_exchange_rate_scd__dbt_tmp_cci + ON test_normalization.dedup_exchange_rate_scd__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..94ab293ce3d4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,122 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."exchange_rate__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate__dbt_tmp"','U') is not null + begin + drop table test_normalization."exchange_rate__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."exchange_rate__dbt_tmp_temp_view" as + +with __dbt__CTE__exchange_rate_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."currency"'') as currency, + json_value(_airbyte_data, ''$."date"'') as "date", + json_value(_airbyte_data, ''$."timestamp_col"'') as timestamp_col, + json_value(_airbyte_data, ''$."HKD@spéçiäl & characters"'') as "HKD@spéçiäl & characters", + json_value(_airbyte_data, ''$."HKD_special___characters"'') as hkd_special___characters, + json_value(_airbyte_data, ''$."NZD"'') as nzd, + json_value(_airbyte_data, ''$."USD"'') as usd, + _airbyte_emitted_at +from "test_normalization".test_normalization._airbyte_raw_exchange_rate as table_alias +-- exchange_rate +), __dbt__CTE__exchange_rate_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + cast(currency as + VARCHAR(max)) as currency, + try_parse("date" as date) as "date", + try_parse(timestamp_col as datetime) as timestamp_col, + cast("HKD@spéçiäl & characters" as + float +) as "HKD@spéçiäl & characters", + cast(hkd_special___characters as + VARCHAR(max)) as hkd_special___characters, + cast(nzd as + float +) as nzd, + cast(usd as + float +) as usd, + _airbyte_emitted_at +from __dbt__CTE__exchange_rate_ab1 +-- exchange_rate +), __dbt__CTE__exchange_rate_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(timestamp_col as + VARCHAR(max)), ''''), ''-'', coalesce(cast("HKD@spéçiäl & characters" as + VARCHAR(max)), ''''), ''-'', coalesce(cast(hkd_special___characters as + VARCHAR(max)), ''''), ''-'', coalesce(cast(nzd as + VARCHAR(max)), ''''), ''-'', coalesce(cast(usd as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, + tmp.* +from __dbt__CTE__exchange_rate_ab2 tmp +-- exchange_rate +)-- Final base SQL model +select + id, + currency, + "date", + timestamp_col, + "HKD@spéçiäl & characters", + hkd_special___characters, + nzd, + usd, + _airbyte_emitted_at, + _airbyte_exchange_rate_hashid +from __dbt__CTE__exchange_rate_ab3 +-- exchange_rate from "test_normalization".test_normalization._airbyte_raw_exchange_rate + '); + + SELECT * INTO "test_normalization".test_normalization."exchange_rate__dbt_tmp" FROM + "test_normalization".test_normalization."exchange_rate__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."exchange_rate__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."exchange_rate__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_exchange_rate__dbt_tmp_cci' + AND object_id=object_id('test_normalization_exchange_rate__dbt_tmp') + ) + DROP index test_normalization.exchange_rate__dbt_tmp.test_normalization_exchange_rate__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_exchange_rate__dbt_tmp_cci + ON test_normalization.exchange_rate__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..c60c7e88ce94 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,100 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" as + +with __dbt__CTE__nested_stream_with_co___long_names_partition_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + json_query("partition", ''$."double_array_data"'') as double_array_data, + json_query("partition", ''$."DATA"'') as "DATA", + json_query("partition", ''$."column`_''''with\"_quotes"'') as "column`_''with""_quotes", + _airbyte_emitted_at +from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" as table_alias +where "partition" is not null +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +), __dbt__CTE__nested_stream_with_co___long_names_partition_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_''with""_quotes", + _airbyte_emitted_at +from __dbt__CTE__nested_stream_with_co___long_names_partition_ab1 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +), __dbt__CTE__nested_stream_with_co___long_names_partition_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_nested_strea__nto_long_names_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(double_array_data as + VARCHAR(max)) as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("DATA" as + VARCHAR(max)) as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast("column`_''with""_quotes" as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_partition_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co___long_names_partition_ab2 tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition +)-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + "DATA", + "column`_''with""_quotes", + _airbyte_emitted_at, + _airbyte_partition_hashid +from __dbt__CTE__nested_stream_with_co___long_names_partition_ab3 +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names" + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___long_names_partition__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp.test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___long_names_partition__dbt_tmp_cci + ON test_normalization.nested_stream_with_co___long_names_partition__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..f8f4c1701ded --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,100 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" as + +with __dbt__CTE__nested_stream_with_co___names_partition_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + "DATA".value, ''$."currency"'') as currency, + _airbyte_emitted_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON("DATA") + ) AS "DATA" +where "DATA" is not null +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +), __dbt__CTE__nested_stream_with_co___names_partition_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + VARCHAR(max)) as currency, + _airbyte_emitted_at +from __dbt__CTE__nested_stream_with_co___names_partition_data_ab1 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +), __dbt__CTE__nested_stream_with_co___names_partition_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_data_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co___names_partition_data_ab2 tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_emitted_at, + _airbyte_data_hashid +from __dbt__CTE__nested_stream_with_co___names_partition_data_ab3 +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co___names_partition_data__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp.test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co___names_partition_data__dbt_tmp_cci + ON test_normalization.nested_stream_with_co___names_partition_data__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql new file mode 100644 index 000000000000..e47b0deb043f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -0,0 +1,100 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" as + +with __dbt__CTE__nested_stream_with_co__column___with__quotes_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + "column`_''with""_quotes".value, ''$."currency"'') as currency, + _airbyte_emitted_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON("column`_''with""_quotes") + ) AS "column`_''with""_quotes" +where "column`_''with""_quotes" is not null +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +), __dbt__CTE__nested_stream_with_co__column___with__quotes_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as + VARCHAR(max)) as currency, + _airbyte_emitted_at +from __dbt__CTE__nested_stream_with_co__column___with__quotes_ab1 +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +), __dbt__CTE__nested_stream_with_co__column___with__quotes_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(currency as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_column___with__quotes_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co__column___with__quotes_ab2 tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes +)-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_emitted_at, + _airbyte_column___with__quotes_hashid +from __dbt__CTE__nested_stream_with_co__column___with__quotes_ab3 +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_''with"_quotes from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__column___with__quotes__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp.test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__column___with__quotes__dbt_tmp_cci + ON test_normalization.nested_stream_with_co__column___with__quotes__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..8aadb9e75291 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,100 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" as + +with __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_partition_hashid, + json_value( + double_array_data.value, ''$."id"'') as id, + _airbyte_emitted_at +from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" as table_alias + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON(double_array_data) + ) AS double_array_data +where double_array_data is not null +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +), __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as + VARCHAR(max)) as id, + _airbyte_emitted_at +from __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab1 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +), __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_partition_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_double_array_data_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab2 tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data +)-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_emitted_at, + _airbyte_double_array_data_hashid +from __dbt__CTE__nested_stream_with_co__ion_double_array_data_ab3 +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "test_normalization".test_normalization."nested_stream_with_co___long_names_partition" + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__ion_double_array_data__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp.test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__ion_double_array_data__dbt_tmp_cci + ON test_normalization.nested_stream_with_co__ion_double_array_data__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql new file mode 100644 index 000000000000..dbb47b24e2ad --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -0,0 +1,57 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" as + +-- Final base SQL model +select + id, + "date", + "partition", + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid +from "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd" +-- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +where _airbyte_active_row = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__lting_into_long_names__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co__lting_into_long_names__dbt_tmp.test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__lting_into_long_names__dbt_tmp_cci + ON test_normalization.nested_stream_with_co__lting_into_long_names__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..05bc3915738e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -0,0 +1,115 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp"','U') is not null + begin + drop table test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" as + +with __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + json_query(_airbyte_data, ''$."partition"'') as "partition", + _airbyte_emitted_at +from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names as table_alias +-- nested_stream_with_co__lting_into_long_names +), __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + cast("partition" as VARCHAR(max)) as "partition", + _airbyte_emitted_at +from __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab1 +-- nested_stream_with_co__lting_into_long_names +), __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''), ''-'', coalesce(cast("partition" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_nested_strea__nto_long_names_hashid, + tmp.* +from __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab2 tmp +-- nested_stream_with_co__lting_into_long_names +), __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab4 as ( + +-- SQL model to prepare for deduplicating records based on the hash record column +select + row_number() over ( + partition by _airbyte_nested_strea__nto_long_names_hashid + order by _airbyte_emitted_at asc + ) as _airbyte_row_num, + tmp.* +from __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab3 tmp +-- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +)-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key +select + id, + "date", + "partition", + "date" as _airbyte_start_at, + lag("date") over ( + partition by id + order by "date" desc, "date" desc, _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag("date") over ( + partition by id + order by "date" desc, "date" desc, _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid +from __dbt__CTE__nested_stream_with_co__lting_into_long_names_ab4 +-- nested_stream_with_co__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +where _airbyte_row_num = 1 + '); + + SELECT * INTO "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp" FROM + "test_normalization".test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_cci' + AND object_id=object_id('test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp') + ) + DROP index test_normalization.nested_stream_with_co__lting_into_long_names_scd__dbt_tmp.test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_nested_stream_with_co__lting_into_long_names_scd__dbt_tmp_cci + ON test_normalization.nested_stream_with_co__lting_into_long_names_scd__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql new file mode 100644 index 000000000000..77f9b1d39063 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -0,0 +1,90 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp"','U') is not null + begin + drop table test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" as + +with __dbt__CTE__non_nested_stream_wit__lting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + _airbyte_emitted_at +from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names as table_alias +-- non_nested_stream_wit__lting_into_long_names +), __dbt__CTE__non_nested_stream_wit__lting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + _airbyte_emitted_at +from __dbt__CTE__non_nested_stream_wit__lting_into_long_names_ab1 +-- non_nested_stream_wit__lting_into_long_names +), __dbt__CTE__non_nested_stream_wit__lting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_non_nested_s__nto_long_names_hashid, + tmp.* +from __dbt__CTE__non_nested_stream_wit__lting_into_long_names_ab2 tmp +-- non_nested_stream_wit__lting_into_long_names +)-- Final base SQL model +select + id, + "date", + _airbyte_emitted_at, + _airbyte_non_nested_s__nto_long_names_hashid +from __dbt__CTE__non_nested_stream_wit__lting_into_long_names_ab3 +-- non_nested_stream_wit__lting_into_long_names from "test_normalization".test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names + '); + + SELECT * INTO "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp" FROM + "test_normalization".test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."non_nested_stream_wit__lting_into_long_names__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci' + AND object_id=object_id('test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp') + ) + DROP index test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp.test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_non_nested_stream_wit__lting_into_long_names__dbt_tmp_cci + ON test_normalization.non_nested_stream_wit__lting_into_long_names__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..2e0f635ec4cf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/unnest_alias.sql @@ -0,0 +1,91 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias__dbt_tmp"','U') is not null + begin + drop table test_normalization."unnest_alias__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."unnest_alias__dbt_tmp_temp_view" as + +with __dbt__CTE__unnest_alias_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_query(_airbyte_data, ''$."children"'') as children, + _airbyte_emitted_at +from "test_normalization".test_normalization._airbyte_raw_unnest_alias as table_alias +-- unnest_alias +), __dbt__CTE__unnest_alias_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + bigint +) as id, + children, + _airbyte_emitted_at +from __dbt__CTE__unnest_alias_ab1 +-- unnest_alias +), __dbt__CTE__unnest_alias_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(cast(children as + VARCHAR(max)) as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_unnest_alias_hashid, + tmp.* +from __dbt__CTE__unnest_alias_ab2 tmp +-- unnest_alias +)-- Final base SQL model +select + id, + children, + _airbyte_emitted_at, + _airbyte_unnest_alias_hashid +from __dbt__CTE__unnest_alias_ab3 +-- unnest_alias from "test_normalization".test_normalization._airbyte_raw_unnest_alias + '); + + SELECT * INTO "test_normalization".test_normalization."unnest_alias__dbt_tmp" FROM + "test_normalization".test_normalization."unnest_alias__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_unnest_alias__dbt_tmp_cci' + AND object_id=object_id('test_normalization_unnest_alias__dbt_tmp') + ) + DROP index test_normalization.unnest_alias__dbt_tmp.test_normalization_unnest_alias__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias__dbt_tmp_cci + ON test_normalization.unnest_alias__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..29b42e18d342 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,106 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp"','U') is not null + begin + drop table test_normalization."unnest_alias_children__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."unnest_alias_children__dbt_tmp_temp_view" as + +with __dbt__CTE__unnest_alias_children_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema + +select + _airbyte_unnest_alias_hashid, + json_value( + children.value, ''$."ab_id"'') as ab_id, + json_query( + children.value, ''$."owner"'') as owner, + _airbyte_emitted_at +from "test_normalization".test_normalization."unnest_alias" as table_alias + + CROSS APPLY ( + SELECT [value] = CASE + WHEN [type] = 4 THEN (SELECT [value] FROM OPENJSON([value])) + WHEN [type] = 5 THEN [value] + END + FROM OPENJSON(children) + ) AS children +where children is not null +-- children at unnest_alias/children +), __dbt__CTE__unnest_alias_children_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as + bigint +) as ab_id, + cast(owner as VARCHAR(max)) as owner, + _airbyte_emitted_at +from __dbt__CTE__unnest_alias_children_ab1 +-- children at unnest_alias/children +), __dbt__CTE__unnest_alias_children_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_unnest_alias_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(ab_id as + VARCHAR(max)), ''''), ''-'', coalesce(cast(owner as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_children_hashid, + tmp.* +from __dbt__CTE__unnest_alias_children_ab2 tmp +-- children at unnest_alias/children +)-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_emitted_at, + _airbyte_children_hashid +from __dbt__CTE__unnest_alias_children_ab3 +-- children at unnest_alias/children from "test_normalization".test_normalization."unnest_alias" + '); + + SELECT * INTO "test_normalization".test_normalization."unnest_alias_children__dbt_tmp" FROM + "test_normalization".test_normalization."unnest_alias_children__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_unnest_alias_children__dbt_tmp_cci' + AND object_id=object_id('test_normalization_unnest_alias_children__dbt_tmp') + ) + DROP index test_normalization.unnest_alias_children__dbt_tmp.test_normalization_unnest_alias_children__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children__dbt_tmp_cci + ON test_normalization.unnest_alias_children__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..04777e88a458 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,91 @@ + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp"','U') is not null + begin + drop table test_normalization."unnest_alias_children_owner__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" as + +with __dbt__CTE__unnest_alias_children_owner_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + json_value(owner, ''$."owner_id"'') as owner_id, + _airbyte_emitted_at +from "test_normalization".test_normalization."unnest_alias_children" as table_alias +where owner is not null +-- owner at unnest_alias/children/owner +), __dbt__CTE__unnest_alias_children_owner_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as + bigint +) as owner_id, + _airbyte_emitted_at +from __dbt__CTE__unnest_alias_children_owner_ab1 +-- owner at unnest_alias/children/owner +), __dbt__CTE__unnest_alias_children_owner_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(_airbyte_children_hashid as + VARCHAR(max)), ''''), ''-'', coalesce(cast(owner_id as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_owner_hashid, + tmp.* +from __dbt__CTE__unnest_alias_children_owner_ab2 tmp +-- owner at unnest_alias/children/owner +)-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_emitted_at, + _airbyte_owner_hashid +from __dbt__CTE__unnest_alias_children_owner_ab3 +-- owner at unnest_alias/children/owner from "test_normalization".test_normalization."unnest_alias_children" + '); + + SELECT * INTO "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp" FROM + "test_normalization".test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization."unnest_alias_children_owner__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_unnest_alias_children_owner__dbt_tmp_cci' + AND object_id=object_id('test_normalization_unnest_alias_children_owner__dbt_tmp') + ) + DROP index test_normalization.unnest_alias_children_owner__dbt_tmp.test_normalization_unnest_alias_children_owner__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_unnest_alias_children_owner__dbt_tmp_cci + ON test_normalization.unnest_alias_children_owner__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql new file mode 100644 index 000000000000..33559361b44b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/final/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -0,0 +1,90 @@ + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + end + + + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp"','U') is not null + begin + drop table test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" + end + + + USE [test_normalization]; + EXEC('create view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" as + +with __dbt__CTE__simple_stream_with_na__lting_into_long_names_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + json_value(_airbyte_data, ''$."id"'') as id, + json_value(_airbyte_data, ''$."date"'') as "date", + _airbyte_emitted_at +from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names as table_alias +-- simple_stream_with_na__lting_into_long_names +), __dbt__CTE__simple_stream_with_na__lting_into_long_names_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as + VARCHAR(max)) as id, + cast("date" as + VARCHAR(max)) as "date", + _airbyte_emitted_at +from __dbt__CTE__simple_stream_with_na__lting_into_long_names_ab1 +-- simple_stream_with_na__lting_into_long_names +), __dbt__CTE__simple_stream_with_na__lting_into_long_names_ab3 as ( + +-- SQL model to build a hash column based on the values of this record +select + convert(varchar(32), HashBytes(''md5'', coalesce(cast( + + + + concat(concat(coalesce(cast(id as + VARCHAR(max)), ''''), ''-'', coalesce(cast("date" as + VARCHAR(max)), ''''),''''), '''') as + VARCHAR(max)), '''')), 2) as _airbyte_simple_strea__nto_long_names_hashid, + tmp.* +from __dbt__CTE__simple_stream_with_na__lting_into_long_names_ab2 tmp +-- simple_stream_with_na__lting_into_long_names +)-- Final base SQL model +select + id, + "date", + _airbyte_emitted_at, + _airbyte_simple_strea__nto_long_names_hashid +from __dbt__CTE__simple_stream_with_na__lting_into_long_names_ab3 +-- simple_stream_with_na__lting_into_long_names from "test_normalization".test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names + '); + + SELECT * INTO "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp" FROM + "test_normalization".test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + + + + USE [test_normalization]; + if object_id ('test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view"','V') is not null + begin + drop view test_normalization_namespace."simple_stream_with_na__lting_into_long_names__dbt_tmp_temp_view" + end + + + use [test_normalization]; + if EXISTS ( + SELECT * FROM + sys.indexes WHERE name = 'test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci' + AND object_id=object_id('test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp') + ) + DROP index test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp.test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci + CREATE CLUSTERED COLUMNSTORE INDEX test_normalization_namespace_simple_stream_with_na__lting_into_long_names__dbt_tmp_cci + ON test_normalization_namespace.simple_stream_with_na__lting_into_long_names__dbt_tmp + + + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/dbt_tmp_data_test/test_tmp_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/dbt_tmp_data_test/test_tmp_data_check_row_counts.sql new file mode 100644 index 000000000000..46fa5def42b6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/dbt_tmp_data_test/test_tmp_data_check_row_counts.sql @@ -0,0 +1,49 @@ +with table_row_counts as ( + select distinct count(*) as row_count, 9 as expected_count + from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +union all + select distinct count(*) as row_count, 9 as expected_count + from {{ ref('exchange_rate') }} + +union all + + select distinct count(*) as row_count, 9 as expected_count + from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +union all + select distinct count(*) as row_count, 9 as expected_count + from {{ ref('dedup_exchange_rate_scd') }} +union all + select distinct count(*) as row_count, 5 as expected_count + from {{ ref('dedup_exchange_rate') }} + +union all + + select distinct count(*) as row_count, 8 as expected_count + from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +union all + select distinct count(*) as row_count, 8 as expected_count + from {{ ref('dedup_cdc_excluded_scd') }} +union all + select distinct count(*) as row_count, 4 as expected_count + from {{ ref('dedup_cdc_excluded') }} + +union all + + select distinct count(*) as row_count, 2 as expected_count + from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +union all + select distinct count(*) as row_count, 2 as expected_count + from {{ ref('nested_stream_with_co__lting_into_long_names') }} +union all + select distinct count(*) as row_count, 2 as expected_count + from {{ ref('nested_stream_with_co___long_names_partition') }} +union all + select count(distinct currency) as row_count, 1 as expected_count + from {{ ref('nested_stream_with_co___names_partition_data') }} +-- union all +-- select count(distinct id) as row_count, 3 as expected_count +-- from {{ ref('nested_stream_with_co__ion_double_array_data') }} +) +select * +from table_row_counts +where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql new file mode 100644 index 000000000000..d3df56ab3d35 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab1.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_array('_airbyte_data', ['conflict_stream_array'], ['conflict_stream_array']) }} as conflict_stream_array, + _airbyte_emitted_at +from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} as table_alias +-- conflict_stream_array + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql new file mode 100644 index 000000000000..2f0da1bb06e7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_string() }}) as id, + conflict_stream_array, + _airbyte_emitted_at +from {{ ref('conflict_stream_array_ab1') }} +-- conflict_stream_array + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql new file mode 100644 index 000000000000..0a92c655f92a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_array_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + array_to_string('conflict_stream_array'), + ]) }} as _airbyte_conflict_stream_array_hashid, + tmp.* +from {{ ref('conflict_stream_array_ab2') }} tmp +-- conflict_stream_array + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..77e7bd31c512 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab1.sql @@ -0,0 +1,10 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_2_hashid, + {{ json_extract_scalar('conflict_stream_name', ['groups'], ['groups']) }} as groups, + _airbyte_emitted_at +from {{ ref('conflict_stream_name_conflict_stream_name') }} as table_alias +where conflict_stream_name is not null +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..ebd4c64d6b0b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_2_hashid, + cast(groups as {{ dbt_utils.type_string() }}) as groups, + _airbyte_emitted_at +from {{ ref('conflict_stream_name____conflict_stream_name_ab1') }} +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..09eaffe2cf54 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name____conflict_stream_name_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + '_airbyte_conflict_stream_name_2_hashid', + 'groups', + ]) }} as _airbyte_conflict_stream_name_3_hashid, + tmp.* +from {{ ref('conflict_stream_name____conflict_stream_name_ab2') }} tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..8556d7f50afb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab1.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract('table_alias', '_airbyte_data', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, + _airbyte_emitted_at +from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} as table_alias +-- conflict_stream_name + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..a1d77f379bce --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_string() }}) as id, + cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, + _airbyte_emitted_at +from {{ ref('conflict_stream_name_ab1') }} +-- conflict_stream_name + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..dc562af89ba1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'conflict_stream_name', + ]) }} as _airbyte_conflict_stream_name_hashid, + tmp.* +from {{ ref('conflict_stream_name_ab2') }} tmp +-- conflict_stream_name + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql new file mode 100644 index 000000000000..d6e2edb75ced --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab1.sql @@ -0,0 +1,10 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_conflict_stream_name_hashid, + {{ json_extract('table_alias', 'conflict_stream_name', ['conflict_stream_name'], ['conflict_stream_name']) }} as conflict_stream_name, + _airbyte_emitted_at +from {{ ref('conflict_stream_name') }} as table_alias +where conflict_stream_name is not null +-- conflict_stream_name at conflict_stream_name/conflict_stream_name + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql new file mode 100644 index 000000000000..463a878933f8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_conflict_stream_name_hashid, + cast(conflict_stream_name as {{ type_json() }}) as conflict_stream_name, + _airbyte_emitted_at +from {{ ref('conflict_stream_name_conflict_stream_name_ab1') }} +-- conflict_stream_name at conflict_stream_name/conflict_stream_name + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql new file mode 100644 index 000000000000..f8d96123ce25 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_name_conflict_stream_name_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + '_airbyte_conflict_stream_name_hashid', + 'conflict_stream_name', + ]) }} as _airbyte_conflict_stream_name_2_hashid, + tmp.* +from {{ ref('conflict_stream_name_conflict_stream_name_ab2') }} tmp +-- conflict_stream_name at conflict_stream_name/conflict_stream_name + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql new file mode 100644 index 000000000000..be57970e399e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab1.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_scalar('_airbyte_data', ['conflict_stream_scalar'], ['conflict_stream_scalar']) }} as conflict_stream_scalar, + _airbyte_emitted_at +from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} as table_alias +-- conflict_stream_scalar + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql new file mode 100644 index 000000000000..cf0d9deb51cd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_string() }}) as id, + cast(conflict_stream_scalar as {{ dbt_utils.type_bigint() }}) as conflict_stream_scalar, + _airbyte_emitted_at +from {{ ref('conflict_stream_scalar_ab1') }} +-- conflict_stream_scalar + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql new file mode 100644 index 000000000000..781c4bbfc391 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/conflict_stream_scalar_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'conflict_stream_scalar', + ]) }} as _airbyte_conflict_stream_scalar_hashid, + tmp.* +from {{ ref('conflict_stream_scalar_ab2') }} tmp +-- conflict_stream_scalar + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql new file mode 100644 index 000000000000..106a78e83df7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab1.sql @@ -0,0 +1,12 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_scalar('_airbyte_data', ['name'], ['name']) }} as name, + {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_lsn'], ['_ab_cdc_lsn']) }} as _ab_cdc_lsn, + {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_updated_at'], ['_ab_cdc_updated_at']) }} as _ab_cdc_updated_at, + {{ json_extract_scalar('_airbyte_data', ['_ab_cdc_deleted_at'], ['_ab_cdc_deleted_at']) }} as _ab_cdc_deleted_at, + _airbyte_emitted_at +from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} as table_alias +-- dedup_cdc_excluded + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql new file mode 100644 index 000000000000..1fe684c4026d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab2.sql @@ -0,0 +1,12 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + cast(name as {{ dbt_utils.type_string() }}) as name, + cast(_ab_cdc_lsn as {{ dbt_utils.type_float() }}) as _ab_cdc_lsn, + cast(_ab_cdc_updated_at as {{ dbt_utils.type_float() }}) as _ab_cdc_updated_at, + cast(_ab_cdc_deleted_at as {{ dbt_utils.type_float() }}) as _ab_cdc_deleted_at, + _airbyte_emitted_at +from {{ ref('dedup_cdc_excluded_ab1') }} +-- dedup_cdc_excluded + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql new file mode 100644 index 000000000000..6df37b07a8cc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab3.sql @@ -0,0 +1,14 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'name', + '_ab_cdc_lsn', + '_ab_cdc_updated_at', + '_ab_cdc_deleted_at', + ]) }} as _airbyte_dedup_cdc_excluded_hashid, + tmp.* +from {{ ref('dedup_cdc_excluded_ab2') }} tmp +-- dedup_cdc_excluded + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql new file mode 100644 index 000000000000..aa34179c3c5c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_cdc_excluded_ab4.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to prepare for deduplicating records based on the hash record column +select + row_number() over ( + partition by _airbyte_dedup_cdc_excluded_hashid + order by _airbyte_emitted_at asc + ) as _airbyte_row_num, + tmp.* +from {{ ref('dedup_cdc_excluded_ab3') }} tmp +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql new file mode 100644 index 000000000000..c9a6865ef7fd --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -0,0 +1,15 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, + {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, + {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, + {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as {{ adapter.quote('HKD@spéçiäl & characters') }}, + {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, + {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, + {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, + _airbyte_emitted_at +from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias +-- dedup_exchange_rate + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql new file mode 100644 index 000000000000..e603d38917b1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -0,0 +1,15 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + cast(currency as {{ dbt_utils.type_string() }}) as currency, + try_parse({{ adapter.quote('date') }} as {{ type_date() }}) as {{ adapter.quote('date') }}, + try_parse(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col, + cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }}, + cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, + cast(nzd as {{ dbt_utils.type_float() }}) as nzd, + cast(usd as {{ dbt_utils.type_float() }}) as usd, + _airbyte_emitted_at +from {{ ref('dedup_exchange_rate_ab1') }} +-- dedup_exchange_rate + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql new file mode 100644 index 000000000000..3e68860804dc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab3.sql @@ -0,0 +1,17 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + adapter.quote('date'), + 'timestamp_col', + adapter.quote('HKD@spéçiäl & characters'), + 'hkd_special___characters', + 'nzd', + 'usd', + ]) }} as _airbyte_dedup_exchange_rate_hashid, + tmp.* +from {{ ref('dedup_exchange_rate_ab2') }} tmp +-- dedup_exchange_rate + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql new file mode 100644 index 000000000000..ef5f6e52d60e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab4.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to prepare for deduplicating records based on the hash record column +select + row_number() over ( + partition by _airbyte_dedup_exchange_rate_hashid + order by _airbyte_emitted_at asc + ) as _airbyte_row_num, + tmp.* +from {{ ref('dedup_exchange_rate_ab3') }} tmp +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql new file mode 100644 index 000000000000..3e66f6d0b4fc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -0,0 +1,15 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, + {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, + {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, + {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as {{ adapter.quote('HKD@spéçiäl & characters') }}, + {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, + {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, + {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, + _airbyte_emitted_at +from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} as table_alias +-- exchange_rate + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql new file mode 100644 index 000000000000..78a747ccf7fc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -0,0 +1,15 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + cast(currency as {{ dbt_utils.type_string() }}) as currency, + try_parse({{ adapter.quote('date') }} as {{ type_date() }}) as {{ adapter.quote('date') }}, + try_parse(timestamp_col as {{ type_timestamp_with_timezone() }}) as timestamp_col, + cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }}, + cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, + cast(nzd as {{ dbt_utils.type_float() }}) as nzd, + cast(usd as {{ dbt_utils.type_float() }}) as usd, + _airbyte_emitted_at +from {{ ref('exchange_rate_ab1') }} +-- exchange_rate + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql new file mode 100644 index 000000000000..962cb57a8666 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -0,0 +1,17 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + 'currency', + adapter.quote('date'), + 'timestamp_col', + adapter.quote('HKD@spéçiäl & characters'), + 'hkd_special___characters', + 'nzd', + 'usd', + ]) }} as _airbyte_exchange_rate_hashid, + tmp.* +from {{ ref('exchange_rate_ab2') }} tmp +-- exchange_rate + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql new file mode 100644 index 000000000000..04ef717d09bf --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab1.sql @@ -0,0 +1,12 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_nested_strea__nto_long_names_hashid, + {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, + {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, + {{ json_extract_array(adapter.quote('partition'), ['column`_\'with"_quotes'], ['column___with__quotes']) }} as {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_emitted_at +from {{ ref('nested_stream_with_co__lting_into_long_names') }} as table_alias +where {{ adapter.quote('partition') }} is not null +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql new file mode 100644 index 000000000000..37d5abd5baff --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab2.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + {{ adapter.quote('DATA') }}, + {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_emitted_at +from {{ ref('nested_stream_with_co___long_names_partition_ab1') }} +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql new file mode 100644 index 000000000000..c9befa862c0a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___long_names_partition_ab3.sql @@ -0,0 +1,13 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + '_airbyte_nested_strea__nto_long_names_hashid', + array_to_string('double_array_data'), + array_to_string(adapter.quote('DATA')), + array_to_string(adapter.quote('column`_\'with""_quotes')), + ]) }} as _airbyte_partition_hashid, + tmp.* +from {{ ref('nested_stream_with_co___long_names_partition_ab2') }} tmp +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql new file mode 100644 index 000000000000..d5c4708f7e07 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab1.sql @@ -0,0 +1,12 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +{{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('DATA')) }} +select + _airbyte_partition_hashid, + {{ json_extract_scalar(unnested_column_value(adapter.quote('DATA')), ['currency'], ['currency']) }} as currency, + _airbyte_emitted_at +from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias +{{ cross_join_unnest('partition', adapter.quote('DATA')) }} +where {{ adapter.quote('DATA') }} is not null +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql new file mode 100644 index 000000000000..bdf11a7ed223 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as {{ dbt_utils.type_string() }}) as currency, + _airbyte_emitted_at +from {{ ref('nested_stream_with_co___names_partition_data_ab1') }} +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql new file mode 100644 index 000000000000..11cdb80699c0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co___names_partition_data_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + '_airbyte_partition_hashid', + 'currency', + ]) }} as _airbyte_data_hashid, + tmp.* +from {{ ref('nested_stream_with_co___names_partition_data_ab2') }} tmp +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql new file mode 100644 index 000000000000..fa16aeba6633 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab1.sql @@ -0,0 +1,12 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +{{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', adapter.quote('column`_\'with""_quotes')) }} +select + _airbyte_partition_hashid, + {{ json_extract_scalar(unnested_column_value(adapter.quote('column`_\'with""_quotes')), ['currency'], ['currency']) }} as currency, + _airbyte_emitted_at +from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias +{{ cross_join_unnest('partition', adapter.quote('column`_\'with""_quotes')) }} +where {{ adapter.quote('column`_\'with""_quotes') }} is not null +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql new file mode 100644 index 000000000000..2f4622e3028a --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(currency as {{ dbt_utils.type_string() }}) as currency, + _airbyte_emitted_at +from {{ ref('nested_stream_with_co__column___with__quotes_ab1') }} +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql new file mode 100644 index 000000000000..62ba655e2e45 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__column___with__quotes_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + '_airbyte_partition_hashid', + 'currency', + ]) }} as _airbyte_column___with__quotes_hashid, + tmp.* +from {{ ref('nested_stream_with_co__column___with__quotes_ab2') }} tmp +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql new file mode 100644 index 000000000000..5b3a0601175b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab1.sql @@ -0,0 +1,12 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +{{ unnest_cte('nested_stream_with_co___long_names_partition', 'partition', 'double_array_data') }} +select + _airbyte_partition_hashid, + {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as id, + _airbyte_emitted_at +from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias +{{ cross_join_unnest('partition', 'double_array_data') }} +where double_array_data is not null +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql new file mode 100644 index 000000000000..e46023b76c26 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_partition_hashid, + cast(id as {{ dbt_utils.type_string() }}) as id, + _airbyte_emitted_at +from {{ ref('nested_stream_with_co__ion_double_array_data_ab1') }} +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql new file mode 100644 index 000000000000..7268452cad18 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__ion_double_array_data_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + '_airbyte_partition_hashid', + 'id', + ]) }} as _airbyte_double_array_data_hashid, + tmp.* +from {{ ref('nested_stream_with_co__ion_double_array_data_ab2') }} tmp +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..e37aca6c48d3 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab1.sql @@ -0,0 +1,10 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, + {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, + _airbyte_emitted_at +from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias +-- nested_stream_with_co__lting_into_long_names + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..68218a656935 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab2.sql @@ -0,0 +1,10 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_string() }}) as id, + cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, + cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, + _airbyte_emitted_at +from {{ ref('nested_stream_with_co__lting_into_long_names_ab1') }} +-- nested_stream_with_co__lting_into_long_names + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..0c9056f13886 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab3.sql @@ -0,0 +1,12 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + adapter.quote('date'), + adapter.quote('partition'), + ]) }} as _airbyte_nested_strea__nto_long_names_hashid, + tmp.* +from {{ ref('nested_stream_with_co__lting_into_long_names_ab2') }} tmp +-- nested_stream_with_co__lting_into_long_names + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql new file mode 100644 index 000000000000..ac961f2763ba --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co__lting_into_long_names_ab4.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to prepare for deduplicating records based on the hash record column +select + row_number() over ( + partition by _airbyte_nested_strea__nto_long_names_hashid + order by _airbyte_emitted_at asc + ) as _airbyte_row_num, + tmp.* +from {{ ref('nested_stream_with_co__lting_into_long_names_ab3') }} tmp +-- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..66f8e4943db1 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab1.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, + _airbyte_emitted_at +from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} as table_alias +-- non_nested_stream_wit__lting_into_long_names + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..ee6798c91bc0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_string() }}) as id, + cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, + _airbyte_emitted_at +from {{ ref('non_nested_stream_wit__lting_into_long_names_ab1') }} +-- non_nested_stream_wit__lting_into_long_names + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..5bdd8ae4c448 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/non_nested_stream_wit__lting_into_long_names_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + adapter.quote('date'), + ]) }} as _airbyte_non_nested_s__nto_long_names_hashid, + tmp.* +from {{ ref('non_nested_stream_wit__lting_into_long_names_ab2') }} tmp +-- non_nested_stream_wit__lting_into_long_names + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql new file mode 100644 index 000000000000..15b34c1ca7d4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab1.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_array('_airbyte_data', ['children'], ['children']) }} as children, + _airbyte_emitted_at +from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} as table_alias +-- unnest_alias + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql new file mode 100644 index 000000000000..a416538b5422 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_bigint() }}) as id, + children, + _airbyte_emitted_at +from {{ ref('unnest_alias_ab1') }} +-- unnest_alias + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql new file mode 100644 index 000000000000..0c6703933b10 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["top-level-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + array_to_string('children'), + ]) }} as _airbyte_unnest_alias_hashid, + tmp.* +from {{ ref('unnest_alias_ab2') }} tmp +-- unnest_alias + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql new file mode 100644 index 000000000000..57abeb23159f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab1.sql @@ -0,0 +1,13 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +{{ unnest_cte('unnest_alias', 'unnest_alias', 'children') }} +select + _airbyte_unnest_alias_hashid, + {{ json_extract_scalar(unnested_column_value('children'), ['ab_id'], ['ab_id']) }} as ab_id, + {{ json_extract('', unnested_column_value('children'), ['owner'], ['owner']) }} as owner, + _airbyte_emitted_at +from {{ ref('unnest_alias') }} as table_alias +{{ cross_join_unnest('unnest_alias', 'children') }} +where children is not null +-- children at unnest_alias/children + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql new file mode 100644 index 000000000000..8db853c1d601 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab2.sql @@ -0,0 +1,10 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_unnest_alias_hashid, + cast(ab_id as {{ dbt_utils.type_bigint() }}) as ab_id, + cast(owner as {{ type_json() }}) as owner, + _airbyte_emitted_at +from {{ ref('unnest_alias_children_ab1') }} +-- children at unnest_alias/children + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql new file mode 100644 index 000000000000..9d84f3be6aa0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_ab3.sql @@ -0,0 +1,12 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + '_airbyte_unnest_alias_hashid', + 'ab_id', + 'owner', + ]) }} as _airbyte_children_hashid, + tmp.* +from {{ ref('unnest_alias_children_ab2') }} tmp +-- children at unnest_alias/children + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql new file mode 100644 index 000000000000..4f28c4f8b5cb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab1.sql @@ -0,0 +1,10 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + _airbyte_children_hashid, + {{ json_extract_scalar('owner', ['owner_id'], ['owner_id']) }} as owner_id, + _airbyte_emitted_at +from {{ ref('unnest_alias_children') }} as table_alias +where owner is not null +-- owner at unnest_alias/children/owner + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql new file mode 100644 index 000000000000..a0f474c62db7 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + _airbyte_children_hashid, + cast(owner_id as {{ dbt_utils.type_bigint() }}) as owner_id, + _airbyte_emitted_at +from {{ ref('unnest_alias_children_owner_ab1') }} +-- owner at unnest_alias/children/owner + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql new file mode 100644 index 000000000000..c5f6562c34c2 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization/unnest_alias_children_owner_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization", tags=["nested-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + '_airbyte_children_hashid', + 'owner_id', + ]) }} as _airbyte_owner_hashid, + tmp.* +from {{ ref('unnest_alias_children_owner_ab2') }} tmp +-- owner at unnest_alias/children/owner + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql new file mode 100644 index 000000000000..1afa8d54d15e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab1.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, + {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, + _airbyte_emitted_at +from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} as table_alias +-- simple_stream_with_na__lting_into_long_names + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql new file mode 100644 index 000000000000..185b0e2f8a39 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab2.sql @@ -0,0 +1,9 @@ +{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +select + cast(id as {{ dbt_utils.type_string() }}) as id, + cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, + _airbyte_emitted_at +from {{ ref('simple_stream_with_na__lting_into_long_names_ab1') }} +-- simple_stream_with_na__lting_into_long_names + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql new file mode 100644 index 000000000000..5766cc587575 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_ctes/test_normalization_namespace/simple_stream_with_na__lting_into_long_names_ab3.sql @@ -0,0 +1,11 @@ +{{ config(schema="_airbyte_test_normalization_namespace", tags=["top-level-intermediate"]) }} +-- SQL model to build a hash column based on the values of this record +select + {{ dbt_utils.surrogate_key([ + 'id', + adapter.quote('date'), + ]) }} as _airbyte_simple_strea__nto_long_names_hashid, + tmp.* +from {{ ref('simple_stream_with_na__lting_into_long_names_ab2') }} tmp +-- simple_stream_with_na__lting_into_long_names + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql new file mode 100644 index 000000000000..9ffda6086e35 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- Final base SQL model +select + id, + conflict_stream_array, + _airbyte_emitted_at, + _airbyte_conflict_stream_array_hashid +from {{ ref('conflict_stream_array_ab3') }} +-- conflict_stream_array from {{ source('test_normalization', '_airbyte_raw_conflict_stream_array') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql new file mode 100644 index 000000000000..8a83ccc01e31 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- Final base SQL model +select + id, + conflict_stream_name, + _airbyte_emitted_at, + _airbyte_conflict_stream_name_hashid +from {{ ref('conflict_stream_name_ab3') }} +-- conflict_stream_name from {{ source('test_normalization', '_airbyte_raw_conflict_stream_name') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql new file mode 100644 index 000000000000..bc23350b4f48 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name____conflict_stream_name.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["nested"]) }} +-- Final base SQL model +select + _airbyte_conflict_stream_name_2_hashid, + groups, + _airbyte_emitted_at, + _airbyte_conflict_stream_name_3_hashid +from {{ ref('conflict_stream_name____conflict_stream_name_ab3') }} +-- conflict_stream_name at conflict_stream_name/conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name_conflict_stream_name') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql new file mode 100644 index 000000000000..94a9dc3d0139 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["nested"]) }} +-- Final base SQL model +select + _airbyte_conflict_stream_name_hashid, + conflict_stream_name, + _airbyte_emitted_at, + _airbyte_conflict_stream_name_2_hashid +from {{ ref('conflict_stream_name_conflict_stream_name_ab3') }} +-- conflict_stream_name at conflict_stream_name/conflict_stream_name from {{ ref('conflict_stream_name') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql new file mode 100644 index 000000000000..9d7acc846fd0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- Final base SQL model +select + id, + conflict_stream_scalar, + _airbyte_emitted_at, + _airbyte_conflict_stream_scalar_hashid +from {{ ref('conflict_stream_scalar_ab3') }} +-- conflict_stream_scalar from {{ source('test_normalization', '_airbyte_raw_conflict_stream_scalar') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql new file mode 100644 index 000000000000..1f229e1cc479 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded.sql @@ -0,0 +1,14 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- Final base SQL model +select + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid +from {{ ref('dedup_cdc_excluded_scd') }} +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +where _airbyte_active_row = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql new file mode 100644 index 000000000000..21910b13b2e6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_cdc_excluded_scd.sql @@ -0,0 +1,23 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key +select + id, + name, + _ab_cdc_lsn, + _ab_cdc_updated_at, + _ab_cdc_deleted_at, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by id + order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag(_airbyte_emitted_at) over ( + partition by id + order by _airbyte_emitted_at desc, _airbyte_emitted_at desc, _airbyte_emitted_at desc, _ab_cdc_updated_at desc + ) is null and _ab_cdc_deleted_at is null then 1 else 0 end as _airbyte_active_row, + _airbyte_emitted_at, + _airbyte_dedup_cdc_excluded_hashid +from {{ ref('dedup_cdc_excluded_ab4') }} +-- dedup_cdc_excluded from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql new file mode 100644 index 000000000000..f10a9446d7b8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate.sql @@ -0,0 +1,17 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- Final base SQL model +select + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid +from {{ ref('dedup_exchange_rate_scd') }} +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +where _airbyte_active_row = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql new file mode 100644 index 000000000000..77ef36c1fe48 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/dedup_exchange_rate_scd.sql @@ -0,0 +1,26 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key +select + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) + order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_emitted_at, + _airbyte_dedup_exchange_rate_hashid +from {{ ref('dedup_exchange_rate_ab4') }} +-- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..147399e6739d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,16 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- Final base SQL model +select + id, + currency, + {{ adapter.quote('date') }}, + timestamp_col, + {{ adapter.quote('HKD@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + _airbyte_emitted_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql new file mode 100644 index 000000000000..463ee8735a5d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___long_names_partition.sql @@ -0,0 +1,12 @@ +{{ config(schema="test_normalization", tags=["nested"]) }} +-- Final base SQL model +select + _airbyte_nested_strea__nto_long_names_hashid, + double_array_data, + {{ adapter.quote('DATA') }}, + {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_emitted_at, + _airbyte_partition_hashid +from {{ ref('nested_stream_with_co___long_names_partition_ab3') }} +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_co__lting_into_long_names') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql new file mode 100644 index 000000000000..9ab923f69627 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["nested"]) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_emitted_at, + _airbyte_data_hashid +from {{ ref('nested_stream_with_co___names_partition_data_ab3') }} +-- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_co___long_names_partition') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql new file mode 100644 index 000000000000..8b126445559b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__column___with__quotes.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["nested"]) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + currency, + _airbyte_emitted_at, + _airbyte_column___with__quotes_hashid +from {{ ref('nested_stream_with_co__column___with__quotes_ab3') }} +-- column___with__quotes at nested_stream_with_complex_columns_resulting_into_long_names/partition/column`_'with"_quotes from {{ ref('nested_stream_with_co___long_names_partition') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql new file mode 100644 index 000000000000..d36f1659ee95 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["nested"]) }} +-- Final base SQL model +select + _airbyte_partition_hashid, + id, + _airbyte_emitted_at, + _airbyte_double_array_data_hashid +from {{ ref('nested_stream_with_co__ion_double_array_data_ab3') }} +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_co___long_names_partition') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql new file mode 100644 index 000000000000..28635caec9e9 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names.sql @@ -0,0 +1,12 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- Final base SQL model +select + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid +from {{ ref('nested_stream_with_co__lting_into_long_names_scd') }} +-- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +where _airbyte_active_row = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql new file mode 100644 index 000000000000..c3ed2b7aad48 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -0,0 +1,21 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key +select + id, + {{ adapter.quote('date') }}, + {{ adapter.quote('partition') }}, + {{ adapter.quote('date') }} as _airbyte_start_at, + lag({{ adapter.quote('date') }}) over ( + partition by id + order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when lag({{ adapter.quote('date') }}) over ( + partition by id + order by {{ adapter.quote('date') }} desc, {{ adapter.quote('date') }} desc, _airbyte_emitted_at desc + ) is null then 1 else 0 end as _airbyte_active_row, + _airbyte_emitted_at, + _airbyte_nested_strea__nto_long_names_hashid +from {{ ref('nested_stream_with_co__lting_into_long_names_ab4') }} +-- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql new file mode 100644 index 000000000000..0cb2abc14b0e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wit__lting_into_long_names.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- Final base SQL model +select + id, + {{ adapter.quote('date') }}, + _airbyte_emitted_at, + _airbyte_non_nested_s__nto_long_names_hashid +from {{ ref('non_nested_stream_wit__lting_into_long_names_ab3') }} +-- non_nested_stream_wit__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql new file mode 100644 index 000000000000..7a570c3836cb --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["top-level"]) }} +-- Final base SQL model +select + id, + children, + _airbyte_emitted_at, + _airbyte_unnest_alias_hashid +from {{ ref('unnest_alias_ab3') }} +-- unnest_alias from {{ source('test_normalization', '_airbyte_raw_unnest_alias') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql new file mode 100644 index 000000000000..77b2b0ad854e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -0,0 +1,11 @@ +{{ config(schema="test_normalization", tags=["nested"]) }} +-- Final base SQL model +select + _airbyte_unnest_alias_hashid, + ab_id, + owner, + _airbyte_emitted_at, + _airbyte_children_hashid +from {{ ref('unnest_alias_children_ab3') }} +-- children at unnest_alias/children from {{ ref('unnest_alias') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql new file mode 100644 index 000000000000..0ccf8e33d8d6 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization", tags=["nested"]) }} +-- Final base SQL model +select + _airbyte_children_hashid, + owner_id, + _airbyte_emitted_at, + _airbyte_owner_hashid +from {{ ref('unnest_alias_children_owner_ab3') }} +-- owner at unnest_alias/children/owner from {{ ref('unnest_alias_children') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql new file mode 100644 index 000000000000..11a61e98ee75 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/airbyte_tables/test_normalization_namespace/simple_stream_with_na__lting_into_long_names.sql @@ -0,0 +1,10 @@ +{{ config(schema="test_normalization_namespace", tags=["top-level"]) }} +-- Final base SQL model +select + id, + {{ adapter.quote('date') }}, + _airbyte_emitted_at, + _airbyte_simple_strea__nto_long_names_hashid +from {{ ref('simple_stream_with_na__lting_into_long_names_ab3') }} +-- simple_stream_with_na__lting_into_long_names from {{ source('test_normalization_namespace', '_airbyte_raw_simple_stream_with_namespace_resulting_into_long_names') }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/sources.yml new file mode 100644 index 000000000000..b3227f513333 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_primary_key_streams/models/generated/sources.yml @@ -0,0 +1,24 @@ +version: 2 +sources: +- name: test_normalization + quoting: + database: true + schema: false + identifier: false + tables: + - name: _airbyte_raw_conflict_stream_array + - name: _airbyte_raw_conflict_stream_name + - name: _airbyte_raw_conflict_stream_scalar + - name: _airbyte_raw_dedup_cdc_excluded + - name: _airbyte_raw_dedup_exchange_rate + - name: _airbyte_raw_exchange_rate + - name: _airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + - name: _airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names + - name: _airbyte_raw_unnest_alias +- name: test_normalization_namespace + quoting: + database: true + schema: false + identifier: false + tables: + - name: _airbyte_raw_simple_stream_with_namespace_resulting_into_long_names diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/data_input/replace_identifiers.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/data_input/replace_identifiers.json index 04063d0702d9..cae288b37802 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/data_input/replace_identifiers.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/data_input/replace_identifiers.json @@ -67,5 +67,24 @@ { "\\\"column`_'with\\\"\\\"_quotes\\\" is not null": "coalesce(json_length(`column__'with\\\"_quotes`), 0) > 0" } + ], + "mssql": [ + { + "nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data": "nested_stream_with_co__ion_double_array_data" + }, + { + "nested_stream_with_complex_columns_resulting_into_long_names_partition_data": "nested_stream_with_co___names_partition_data" + }, + { + "nested_stream_with_complex_columns_resulting_into_long_names_partition": "nested_stream_with_co___long_names_partition" + }, + { + "'nested_stream_with_complex_columns_resulting_into_long_names'": "'nested_stream_with_co__lting_into_long_names'" + }, + { + "non_nested_stream_without_namespace_resulting_into_long_names": "non_nested_stream_wit__lting_into_long_names" + }, + { "- HKD_special___characters": "- '\"HKD_special___characters\"'" }, + { "!= HKD_special___characters": "!= \"HKD_special___characters\"" } ] } diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/dbt_data_tests_mssql/test_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/dbt_data_tests_mssql/test_check_row_counts.sql new file mode 100644 index 000000000000..0a92e68f63cc --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/dbt_data_tests_mssql/test_check_row_counts.sql @@ -0,0 +1 @@ +select * from {{ ref('test_tmp_data_check_row_counts') }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/dbt_tmp_data_test_mssql/test_tmp_data_check_row_counts.sql b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/dbt_tmp_data_test_mssql/test_tmp_data_check_row_counts.sql new file mode 100644 index 000000000000..d4958111f483 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_primary_key_streams/dbt_tmp_data_test_mssql/test_tmp_data_check_row_counts.sql @@ -0,0 +1,49 @@ +with table_row_counts as ( + select distinct count(*) as row_count, 9 as expected_count + from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +union all + select distinct count(*) as row_count, 9 as expected_count + from {{ ref('exchange_rate') }} + +union all + + select distinct count(*) as row_count, 9 as expected_count + from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +union all + select distinct count(*) as row_count, 9 as expected_count + from {{ ref('dedup_exchange_rate_scd') }} +union all + select distinct count(*) as row_count, 5 as expected_count + from {{ ref('dedup_exchange_rate') }} + +union all + + select distinct count(*) as row_count, 8 as expected_count + from {{ source('test_normalization', '_airbyte_raw_dedup_cdc_excluded') }} +union all + select distinct count(*) as row_count, 8 as expected_count + from {{ ref('dedup_cdc_excluded_scd') }} +union all + select distinct count(*) as row_count, 4 as expected_count + from {{ ref('dedup_cdc_excluded') }} + +union all + + select distinct count(*) as row_count, 2 as expected_count + from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +union all + select distinct count(*) as row_count, 2 as expected_count + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names') }} +union all + select distinct count(*) as row_count, 2 as expected_count + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition') }} +union all + select count(distinct currency) as row_count, 1 as expected_count + from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_DATA') }} +-- union all +-- select count(distinct id) as row_count, 3 as expected_count +-- from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data') }} +) +select * +from table_row_counts +where row_count != expected_count diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py b/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py index 792731b6e58b..f167c93128ca 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py @@ -49,6 +49,10 @@ def test_destination_supported_limits(integration_type: DestinationType, column_ return if integration_type == DestinationType.ORACLE: column_count = 998 + if integration_type == DestinationType.MSSQL: + # In MS SQL Server, the max number of columns / table = 1024, + # We should leave the space for the '_airbyte_emitted_at' column. So 1022 is the max what could be inserted. + column_count = 1022 run_test(integration_type, column_count) @@ -56,19 +60,12 @@ def test_destination_supported_limits(integration_type: DestinationType, column_ "integration_type, column_count, expected_exception_message", [ ("Postgres", 1665, "target lists can have at most 1664 entries"), - ( - "BigQuery", - 2500, - "The view is too large.", - ), - ( - "Snowflake", - 2000, - "Operation failed because soft limit on objects of type 'Column' per table was exceeded.", - ), + ("BigQuery", 2500, "The view is too large."), + ("Snowflake", 2000, "Operation failed because soft limit on objects of type 'Column' per table was exceeded."), ("Redshift", 1665, "target lists can have at most 1664 entries"), ("MySQL", 250, "Row size too large"), ("Oracle", 1001, "ORA-01792: maximum number of columns in a table or view is 1000"), + ("MSSQL", 1025, "exceeds the maximum of 1024 columns."), ], ) def test_destination_failure_over_limits(integration_type: str, column_count: int, expected_exception_message: str, setup_test_path): diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py b/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py index bc84e6e76267..2f8e2321c73a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py @@ -55,26 +55,34 @@ def setup_test_path(request): ), ) # Uncomment the following line as an example on how to run the test against local destinations only... -# @pytest.mark.parametrize("destination_type", [DestinationType.POSTGRES, DestinationType.MYSQL]) +# @pytest.mark.parametrize("destination_type", [DestinationType.POSTGRES, DestinationType.MYSQL, ...]) # Run tests on all destinations: @pytest.mark.parametrize("destination_type", list(DestinationType)) def test_normalization(destination_type: DestinationType, test_resource_name: str, setup_test_path): print("Testing normalization") integration_type = destination_type.value + # Create the test folder with dbt project and appropriate destination settings to run integration tests from test_root_dir = setup_test_dir(integration_type, test_resource_name) destination_config = dbt_test_utils.generate_profile_yaml_file(destination_type, test_root_dir) dbt_test_utils.generate_project_yaml_file(destination_type, test_root_dir) + # Use destination connector to create _airbyte_raw_* tables to use as input for the test assert setup_input_raw_data(integration_type, test_resource_name, test_root_dir, destination_config) + # Normalization step generate_dbt_models(destination_type, test_resource_name, test_root_dir) - dbt_test_utils.dbt_run(test_root_dir) - - if integration_type != DestinationType.ORACLE.value: + if integration_type == DestinationType.ORACLE.value: # Oracle doesnt support nested with clauses + # Skip the dbt_test for DestinationType.ORACLE + dbt_test_utils.dbt_run(test_root_dir) + else: + # Setup test resources and models + dbt_test_setup(destination_type, test_resource_name, test_root_dir) + # Run DBT process + dbt_test_utils.dbt_run(test_root_dir) # Run checks on Tests results - dbt_test(destination_type, test_resource_name, test_root_dir) + dbt_test(test_root_dir) check_outputs(destination_type, test_resource_name, test_root_dir) @@ -164,28 +172,52 @@ def generate_dbt_models(destination_type: DestinationType, test_resource_name: s ) -def dbt_test(destination_type: DestinationType, test_resource_name: str, test_root_dir: str): +def dbt_test_setup(destination_type: DestinationType, test_resource_name: str, test_root_dir: str): """ - dbt provides a way to run dbt tests as described here: https://docs.getdbt.com/docs/building-a-dbt-project/tests - - Schema tests are added in .yml files from the schema_tests directory - - see additional macros for testing here: https://github.com/fishtown-analytics/dbt-utils#schema-tests - - Data tests are added in .sql files from the data_tests directory and should return 0 records to be successful - - We use this mechanism to verify the output of our integration tests. + Prepare the data (copy) for the models for dbt test. """ replace_identifiers = os.path.join("resources", test_resource_name, "data_input", "replace_identifiers.json") + + # COMMON TEST RESOURCES copy_test_files( os.path.join("resources", test_resource_name, "dbt_schema_tests"), os.path.join(test_root_dir, "models/dbt_schema_tests"), destination_type, replace_identifiers, ) - copy_test_files( - os.path.join("resources", test_resource_name, "dbt_data_tests"), - os.path.join(test_root_dir, "tests"), - destination_type, - replace_identifiers, - ) + if destination_type.value == DestinationType.MSSQL.value: + # MS SQL Server doesn't support nested WITH clause, + # we use the separed sql statement, to handle this + copy_test_files( + os.path.join("resources", test_resource_name, "dbt_data_tests_mssql"), + os.path.join(test_root_dir, "tests"), + destination_type, + replace_identifiers, + ) + copy_test_files( + os.path.join("resources", test_resource_name, "dbt_tmp_data_test_mssql"), + os.path.join(test_root_dir, "models/dbt_tmp_data_test"), + destination_type, + replace_identifiers, + ) + else: + copy_test_files( + os.path.join("resources", test_resource_name, "dbt_data_tests"), + os.path.join(test_root_dir, "tests"), + destination_type, + replace_identifiers, + ) + + +def dbt_test(test_root_dir: str): + """ + dbt provides a way to run dbt tests as described here: https://docs.getdbt.com/docs/building-a-dbt-project/tests + - Schema tests are added in .yml files from the schema_tests directory + - see additional macros for testing here: https://github.com/fishtown-analytics/dbt-utils#schema-tests + - Data tests are added in .sql files from the data_tests directory and should return 0 records to be successful + + We use this mechanism to verify the output of our integration tests. + """ assert dbt_test_utils.run_check_dbt_command("test", test_root_dir) diff --git a/airbyte-integrations/bases/base-normalization/normalization/destination_type.py b/airbyte-integrations/bases/base-normalization/normalization/destination_type.py index 951b1878d97f..be1f76df02b6 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/destination_type.py +++ b/airbyte-integrations/bases/base-normalization/normalization/destination_type.py @@ -13,6 +13,7 @@ class DestinationType(Enum): SNOWFLAKE = "snowflake" MYSQL = "mysql" ORACLE = "oracle" + MSSQL = "mssql" @classmethod def from_string(cls, string_value: str) -> "DestinationType": diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/destination_name_transformer.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/destination_name_transformer.py index 6650a1b909be..ac4609626159 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/destination_name_transformer.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/destination_name_transformer.py @@ -23,6 +23,8 @@ DestinationType.MYSQL.value: 64, # https://oracle-base.com/articles/12c/long-identifiers-12cr2 DestinationType.ORACLE.value: 128, + # https://docs.microsoft.com/en-us/sql/odbc/microsoft/column-name-limitations?view=sql-server-ver15 + DestinationType.MSSQL.value: 64, } # DBT also needs to generate suffix to table names, so we need to make sure it has enough characters to do so... @@ -203,6 +205,9 @@ def __normalize_identifier_case(self, input_name: str, is_quoted: bool = False) elif self.destination_type.value == DestinationType.MYSQL.value: if not is_quoted and not self.needs_quotes(input_name): result = input_name.lower() + elif self.destination_type.value == DestinationType.MSSQL.value: + if not is_quoted and not self.needs_quotes(input_name): + result = input_name.lower() elif self.destination_type.value == DestinationType.ORACLE.value: if not is_quoted and not self.needs_quotes(input_name): result = input_name.lower() diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/reserved_keywords.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/reserved_keywords.py index 911527ded3a3..7f69a3610883 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/reserved_keywords.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/reserved_keywords.py @@ -2045,6 +2045,494 @@ "WITH", } + +# https://docs.microsoft.com/en-us/sql/t-sql/language-elements/reserved-keywords-transact-sql?view=sql-server-ver15 +MSSQL = { + "ADD", + "EXTERNAL", + "PROCEDURE", + "ALL", + "FETCH", + "PUBLIC", + "ALTER", + "FILE", + "RAISERROR", + "AND", + "FILLFACTOR", + "READ", + "ANY", + "FOR", + "READTEXT", + "AS", + "FOREIGN", + "RECONFIGURE", + "ASC", + "FREETEXT", + "REFERENCES", + "AUTHORIZATION", + "FREETEXTTABLE", + "REPLICATION", + "BACKUP", + "FROM", + "RESTORE", + "BEGIN", + "FULL", + "RESTRICT", + "BETWEEN", + "FUNCTION", + "RETURN", + "BREAK", + "GOTO", + "REVERT", + "BROWSE", + "GRANT", + "REVOKE", + "BULK", + "GROUP", + "RIGHT", + "BY", + "HAVING", + "ROLLBACK", + "CASCADE", + "HOLDLOCK", + "ROWCOUNT", + "CASE", + "IDENTITY", + "ROWGUIDCOL", + "CHECK", + "IDENTITY_INSERT", + "RULE", + "CHECKPOINT", + "IDENTITYCOL", + "SAVE", + "CLOSE", + "IF", + "SCHEMA", + "CLUSTERED", + "IN", + "SECURITYAUDIT", + "COALESCE", + "INDEX", + "SELECT", + "COLLATE", + "INNER", + "SEMANTICKEYPHRASETABLE", + "COLUMN", + "INSERT", + "SEMANTICSIMILARITYDETAILSTABLE", + "COMMIT", + "INTERSECT", + "SEMANTICSIMILARITYTABLE", + "COMPUTE", + "INTO", + "SESSION_USER", + "CONSTRAINT", + "IS", + "SET", + "CONTAINS", + "JOIN", + "SETUSER", + "CONTAINSTABLE", + "KEY", + "SHUTDOWN", + "CONTINUE", + "KILL", + "SOME", + "CONVERT", + "LEFT", + "STATISTICS", + "CREATE", + "LIKE", + "SYSTEM_USER", + "CROSS", + "LINENO", + "TABLE", + "CURRENT", + "LOAD", + "TABLESAMPLE", + "CURRENT_DATE", + "MERGE", + "TEXTSIZE", + "CURRENT_TIME", + "NATIONAL", + "THEN", + "CURRENT_TIMESTAMP", + "NOCHECK", + "TO", + "CURRENT_USER", + "NONCLUSTERED", + "TOP", + "CURSOR", + "NOT", + "TRAN", + "DATABASE", + "NULL", + "TRANSACTION", + "DBCC", + "NULLIF", + "TRIGGER", + "DEALLOCATE", + "OF", + "TRUNCATE", + "DECLARE", + "OFF", + "TRY_CONVERT", + "DEFAULT", + "OFFSETS", + "TSEQUAL", + "DELETE", + "ON", + "UNION", + "DENY", + "OPEN", + "UNIQUE", + "DESC", + "OPENDATASOURCE", + "UNPIVOT", + "DISK", + "OPENQUERY", + "UPDATE", + "DISTINCT", + "OPENROWSET", + "UPDATETEXT", + "DISTRIBUTED", + "OPENXML", + "USE", + "DOUBLE", + "OPTION", + "USER", + "DROP", + "OR", + "VALUES", + "DUMP", + "ORDER", + "VARYING", + "ELSE", + "OUTER", + "VIEW", + "END", + "OVER", + "WAITFOR", + "ERRLVL", + "PERCENT", + "WHEN", + "ESCAPE", + "PIVOT", + "WHERE", + "EXCEPT", + "PLAN", + "WHILE", + "EXEC", + "PRECISION", + "WITH", + "EXECUTE", + "PRIMARY", + "WITHIN GROUP", + "EXISTS", + "PRINT", + "WRITETEXT", + "EXIT", + "PROC", + "ABSOLUTE", + "OVERLAPS", + "ACTION", + "PAD", + "ADA", + "PARTIAL", + "PASCAL", + "EXTRACT", + "POSITION", + "ALLOCATE", + "FALSE", + "PREPARE", + "FIRST", + "PRESERVE", + "FLOAT", + "ARE", + "PRIOR", + "PRIVILEGES", + "FORTRAN", + "ASSERTION", + "FOUND", + "AT", + "REAL", + "AVG", + "GET", + "GLOBAL", + "RELATIVE", + "GO", + "BIT", + "BIT_LENGTH", + "BOTH", + "ROWS", + "HOUR", + "CASCADED", + "SCROLL", + "IMMEDIATE", + "SECOND", + "CAST", + "SECTION", + "CATALOG", + "INCLUDE", + "CHAR", + "SESSION", + "CHAR_LENGTH", + "INDICATOR", + "CHARACTER", + "INITIALLY", + "CHARACTER_LENGTH", + "SIZE", + "INPUT", + "SMALLINT", + "INSENSITIVE", + "SPACE", + "INT", + "SQL", + "COLLATION", + "INTEGER", + "SQLCA", + "SQLCODE", + "INTERVAL", + "SQLERROR", + "CONNECT", + "SQLSTATE", + "CONNECTION", + "SQLWARNING", + "ISOLATION", + "SUBSTRING", + "CONSTRAINTS", + "SUM", + "LANGUAGE", + "CORRESPONDING", + "LAST", + "TEMPORARY", + "COUNT", + "LEADING", + "TIME", + "LEVEL", + "TIMESTAMP", + "TIMEZONE_HOUR", + "LOCAL", + "TIMEZONE_MINUTE", + "LOWER", + "MATCH", + "TRAILING", + "MAX", + "MIN", + "TRANSLATE", + "DATE", + "MINUTE", + "TRANSLATION", + "DAY", + "MODULE", + "TRIM", + "MONTH", + "TRUE", + "DEC", + "NAMES", + "DECIMAL", + "NATURAL", + "UNKNOWN", + "NCHAR", + "DEFERRABLE", + "NEXT", + "UPPER", + "DEFERRED", + "NO", + "USAGE", + "NONE", + "USING", + "DESCRIBE", + "VALUE", + "DESCRIPTOR", + "DIAGNOSTICS", + "NUMERIC", + "VARCHAR", + "DISCONNECT", + "OCTET_LENGTH", + "DOMAIN", + "ONLY", + "WHENEVER", + "WORK", + "END-EXEC", + "WRITE", + "YEAR", + "OUTPUT", + "ZONE", + "EXCEPTION", + "HOST", + "RELEASE", + "ADMIN", + "IGNORE", + "RESULT", + "AFTER", + "RETURNS", + "AGGREGATE", + "ROLE", + "ALIAS", + "INITIALIZE", + "ROLLUP", + "ROUTINE", + "INOUT", + "ROW", + "ARRAY", + "ASENSITIVE", + "SAVEPOINT", + "ASYMMETRIC", + "INTERSECTION", + "SCOPE", + "SEARCH", + "ATOMIC", + "BEFORE", + "ITERATE", + "BINARY", + "SENSITIVE", + "LARGE", + "SEQUENCE", + "BLOB", + "BOOLEAN", + "LATERAL", + "SETS", + "SIMILAR", + "BREADTH", + "LESS", + "CALL", + "CALLED", + "LIKE_REGEX", + "CARDINALITY", + "LIMIT", + "SPECIFIC", + "LN", + "SPECIFICTYPE", + "LOCALTIME", + "SQLEXCEPTION", + "LOCALTIMESTAMP", + "LOCATOR", + "CLASS", + "MAP", + "START", + "CLOB", + "STATE", + "MEMBER", + "STATEMENT", + "COLLECT", + "METHOD", + "STATIC", + "COMPLETION", + "STDDEV_POP", + "CONDITION", + "MOD", + "STDDEV_SAMP", + "MODIFIES", + "STRUCTURE", + "MODIFY", + "SUBMULTISET", + "SUBSTRING_REGEX", + "CONSTRUCTOR", + "SYMMETRIC", + "CORR", + "MULTISET", + "SYSTEM", + "COVAR_POP", + "TERMINATE", + "COVAR_SAMP", + "THAN", + "CUBE", + "NCLOB", + "CUME_DIST", + "NEW", + "CURRENT_CATALOG", + "CURRENT_DEFAULT_TRANSFORM_GROUP", + "CURRENT_PATH", + "CURRENT_ROLE", + "NORMALIZE", + "TRANSLATE_REGEX", + "CURRENT_SCHEMA", + "CURRENT_TRANSFORM_GROUP_FOR_TYPE", + "OBJECT", + "TREAT", + "CYCLE", + "OCCURRENCES_REGEX", + "DATA", + "OLD", + "UESCAPE", + "UNDER", + "OPERATION", + "ORDINALITY", + "UNNEST", + "OUT", + "OVERLAY", + "DEPTH", + "VAR_POP", + "DEREF", + "PARAMETER", + "VAR_SAMP", + "PARAMETERS", + "VARIABLE", + "DESTROY", + "PARTITION", + "DESTRUCTOR", + "PATH", + "WIDTH_BUCKET", + "DETERMINISTIC", + "POSTFIX", + "WITHOUT", + "DICTIONARY", + "PREFIX", + "WINDOW", + "PREORDER", + "WITHIN", + "PERCENT_RANK", + "DYNAMIC", + "PERCENTILE_CONT", + "XMLAGG", + "EACH", + "PERCENTILE_DISC", + "XMLATTRIBUTES", + "ELEMENT", + "POSITION_REGEX", + "XMLBINARY", + "XMLCAST", + "EQUALS", + "XMLCOMMENT", + "EVERY", + "XMLCONCAT", + "RANGE", + "XMLDOCUMENT", + "READS", + "XMLELEMENT", + "FILTER", + "XMLEXISTS", + "RECURSIVE", + "XMLFOREST", + "REF", + "XMLITERATE", + "REFERENCING", + "XMLNAMESPACES", + "FREE", + "REGR_AVGX", + "XMLPARSE", + "FULLTEXTTABLE", + "REGR_AVGY", + "XMLPI", + "FUSION", + "REGR_COUNT", + "XMLQUERY", + "GENERAL", + "REGR_INTERCEPT", + "XMLSERIALIZE", + "REGR_R2", + "XMLTABLE", + "REGR_SLOPE", + "XMLTEXT", + "REGR_SXX", + "XMLVALIDATE", + "GROUPING", + "REGR_SXY", + "HOLD", + "REGR_SYY", +} + RESERVED_KEYWORDS = { DestinationType.BIGQUERY.value: BIGQUERY, DestinationType.POSTGRES.value: POSTGRES, @@ -2052,6 +2540,7 @@ DestinationType.SNOWFLAKE.value: SNOWFLAKE, DestinationType.MYSQL.value: MYSQL, DestinationType.ORACLE.value: ORACLE, + DestinationType.MSSQL.value: MSSQL, } diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 5be264737bd7..46b74c0ac95e 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -419,9 +419,18 @@ def cast_property_type(self, property_name: str, column_name: str, jinja_column: # snowflake uses case when statement to parse timestamp field # in this case [cast] operator is not needed as data already converted to timestamp type return self.generate_snowflake_timestamp_statement(column_name) + if self.destination_type == DestinationType.MSSQL: + # in case of datetime, we don't need to use [cast] function, use try_parse instead. + sql_type = jinja_call("type_timestamp_with_timezone()") + return f"try_parse({column_name} as {sql_type}) as {column_name}" + # in all other cases sql_type = jinja_call("type_timestamp_with_timezone()") - elif is_date(definition): + if self.destination_type == DestinationType.MSSQL: + # in case of date, we don't need to use [cast] function, use try_parse instead. + sql_type = jinja_call("type_date()") + return f"try_parse({column_name} as {sql_type}) as {column_name}" + # in all other cases sql_type = jinja_call("type_date()") elif is_string(definition["type"]): sql_type = jinja_call("dbt_utils.type_string()") @@ -576,6 +585,9 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup order_null = "is null asc" if self.destination_type == DestinationType.ORACLE: order_null = "asc nulls first" + if self.destination_type == DestinationType.MSSQL: + # SQL Server treats NULL values as the lowest values, then sorted in ascending order, NULLs come first. + order_null = "desc" cdc_active_row_pattern = "" cdc_updated_order_pattern = "" diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py index 3016761a329e..34b5bc3b29cc 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_config/transform.py @@ -21,6 +21,7 @@ class DestinationType(Enum): snowflake = "snowflake" mysql = "mysql" oracle = "oracle" + mssql = "mssql" class TransformConfig: @@ -80,6 +81,7 @@ def transform(self, integration_type: DestinationType, config: Dict[str, Any]): DestinationType.snowflake.value: self.transform_snowflake, DestinationType.mysql.value: self.transform_mysql, DestinationType.oracle.value: self.transform_oracle, + DestinationType.mssql.value: self.transform_mssql, }[integration_type.value](config) # merge pre-populated base_profile with destination-specific configuration. @@ -257,6 +259,25 @@ def transform_oracle(config: Dict[str, Any]): } return dbt_config + @staticmethod + def transform_mssql(config: Dict[str, Any]): + print("transform_mssql") + # https://docs.getdbt.com/reference/warehouse-profiles/mssql-profile + dbt_config = { + "type": "sqlserver", + "driver": "ODBC Driver 17 for SQL Server", + "server": config["host"], + "port": config["port"], + "schema": config["schema"], + "database": config["database"], + "user": config["username"], + "password": config["password"], + "threads": 32, + # "authentication": "sql", + # "trusted_connection": True, + } + return dbt_config + @staticmethod def read_json_config(input_path: str): with open(input_path, "r") as file: diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/resources/long_name_truncate_collisions_catalog_expected_mssql_names.json b/airbyte-integrations/bases/base-normalization/unit_tests/resources/long_name_truncate_collisions_catalog_expected_mssql_names.json new file mode 100644 index 000000000000..160fc5b70b75 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/unit_tests/resources/long_name_truncate_collisions_catalog_expected_mssql_names.json @@ -0,0 +1,32 @@ +{ + "_airbyte_another.postgres_has_a_64_characters_limit_to_table_names_but_other_destinations_are_fine.postgres_has_a_64_characters_limit_to_table_names_but_other_destinations_are_fine": { + "file": "postgres_has_a_64_cha__destinations_are_fine", + "schema": "_airbyte_another", + "table": "postgres_has_a_64_cha__destinations_are_fine" + }, + "_airbyte_schema_test.postgres_has_a_64_characters_and_not_more_limit_to_table_names_but_other_destinations_are_fine.postgres_has_a_64_characters_and_not_more_limit_to_table_names_but_other_destinations_are_fine": { + "file": "postgres_has_a_64_cha__inations_are_fine_d2b", + "schema": "_airbyte_schema_test", + "table": "postgres_has_a_64_cha__inations_are_fine_d2b" + }, + "_airbyte_schema_test.postgres_has_a_64_characters_limit_to_table_names_but_other_destinations_are_fine.postgres_has_a_64_characters_limit_to_table_names_but_other_destinations_are_fine": { + "file": "postgres_has_a_64_cha__inations_are_fine_e5a", + "schema": "_airbyte_schema_test", + "table": "postgres_has_a_64_cha__inations_are_fine_e5a" + }, + "another.postgres_has_a_64_characters_limit_to_table_names_but_other_destinations_are_fine.postgres_has_a_64_characters_limit_to_table_names_but_other_destinations_are_fine": { + "file": "postgres_has_a_64_cha__destinations_are_fine", + "schema": "another", + "table": "postgres_has_a_64_cha__destinations_are_fine" + }, + "schema_test.postgres_has_a_64_characters_and_not_more_limit_to_table_names_but_other_destinations_are_fine.postgres_has_a_64_characters_and_not_more_limit_to_table_names_but_other_destinations_are_fine": { + "file": "postgres_has_a_64_cha__inations_are_fine_d2b", + "schema": "schema_test", + "table": "postgres_has_a_64_cha__inations_are_fine_d2b" + }, + "schema_test.postgres_has_a_64_characters_limit_to_table_names_but_other_destinations_are_fine.postgres_has_a_64_characters_limit_to_table_names_but_other_destinations_are_fine": { + "file": "postgres_has_a_64_cha__inations_are_fine_e5a", + "schema": "schema_test", + "table": "postgres_has_a_64_cha__inations_are_fine_e5a" + } +} diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/resources/nested_catalog_expected_mssql_names.json b/airbyte-integrations/bases/base-normalization/unit_tests/resources/nested_catalog_expected_mssql_names.json new file mode 100644 index 000000000000..2bbb864cc4d8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/unit_tests/resources/nested_catalog_expected_mssql_names.json @@ -0,0 +1,252 @@ +{ + "_airbyte_schema_test.adcreatives.adcreatives": { + "file": "adcreatives", + "schema": "_airbyte_schema_test", + "table": "adcreatives" + }, + "_airbyte_schema_test.adcreatives_adlabels.adlabels": { + "file": "adcreatives_adlabels", + "schema": "_airbyte_schema_test", + "table": "adcreatives_adlabels" + }, + "_airbyte_schema_test.adcreatives_image_crops.image_crops": { + "file": "adcreatives_image_crops", + "schema": "_airbyte_schema_test", + "table": "adcreatives_image_crops" + }, + "_airbyte_schema_test.adcreatives_image_crops_100x100.100x100": { + "file": "adcreatives_image_crops_100x100", + "schema": "_airbyte_schema_test", + "table": "adcreatives_image_crops_100x100" + }, + "_airbyte_schema_test.adcreatives_image_crops_100x72.100x72": { + "file": "adcreatives_image_crops_100x72", + "schema": "_airbyte_schema_test", + "table": "adcreatives_image_crops_100x72" + }, + "_airbyte_schema_test.adcreatives_image_crops_191x100.191x100": { + "file": "adcreatives_image_crops_191x100", + "schema": "_airbyte_schema_test", + "table": "adcreatives_image_crops_191x100" + }, + "_airbyte_schema_test.adcreatives_image_crops_400x150.400x150": { + "file": "adcreatives_image_crops_400x150", + "schema": "_airbyte_schema_test", + "table": "adcreatives_image_crops_400x150" + }, + "_airbyte_schema_test.adcreatives_image_crops_400x500.400x500": { + "file": "adcreatives_image_crops_400x500", + "schema": "_airbyte_schema_test", + "table": "adcreatives_image_crops_400x500" + }, + "_airbyte_schema_test.adcreatives_image_crops_600x360.600x360": { + "file": "adcreatives_image_crops_600x360", + "schema": "_airbyte_schema_test", + "table": "adcreatives_image_crops_600x360" + }, + "_airbyte_schema_test.adcreatives_image_crops_90x160.90x160": { + "file": "adcreatives_image_crops_90x160", + "schema": "_airbyte_schema_test", + "table": "adcreatives_image_crops_90x160" + }, + "_airbyte_schema_test.adcreatives_object_story_spec.object_story_spec": { + "file": "adcreatives_object_story_spec", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_story_spec" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data.link_data": { + "file": "adcreatives_object_story_spec_link_data", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_story_spec_link_data" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_app_link_spec.app_link_spec": { + "file": "adcreatives_object_st__nk_data_app_link_spec", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__nk_data_app_link_spec" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_app_link_spec_android.android": { + "file": "adcreatives_object_st__app_link_spec_android", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__app_link_spec_android" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_app_link_spec_ios.ios": { + "file": "adcreatives_object_st__ata_app_link_spec_ios", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__ata_app_link_spec_ios" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_app_link_spec_ipad.ipad": { + "file": "adcreatives_object_st__ta_app_link_spec_ipad", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__ta_app_link_spec_ipad" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_app_link_spec_iphone.iphone": { + "file": "adcreatives_object_st___app_link_spec_iphone", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st___app_link_spec_iphone" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_image_crops.image_crops": { + "file": "adcreatives_object_st__link_data_image_crops", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__link_data_image_crops" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_image_crops_100x100.100x100": { + "file": "adcreatives_object_st__a_image_crops_100x100", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__a_image_crops_100x100" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_image_crops_100x72.100x72": { + "file": "adcreatives_object_st__ta_image_crops_100x72", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__ta_image_crops_100x72" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_image_crops_191x100.191x100": { + "file": "adcreatives_object_st__a_image_crops_191x100", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__a_image_crops_191x100" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_image_crops_400x150.400x150": { + "file": "adcreatives_object_st__a_image_crops_400x150", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__a_image_crops_400x150" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_image_crops_400x500.400x500": { + "file": "adcreatives_object_st__a_image_crops_400x500", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__a_image_crops_400x500" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_image_crops_600x360.600x360": { + "file": "adcreatives_object_st__a_image_crops_600x360", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__a_image_crops_600x360" + }, + "_airbyte_schema_test.adcreatives_object_story_spec_link_data_image_crops_90x160.90x160": { + "file": "adcreatives_object_st__ta_image_crops_90x160", + "schema": "_airbyte_schema_test", + "table": "adcreatives_object_st__ta_image_crops_90x160" + }, + "schema_test.adcreatives.adcreatives": { + "file": "adcreatives", + "schema": "schema_test", + "table": "adcreatives" + }, + "schema_test.adcreatives_adlabels.adlabels": { + "file": "adcreatives_adlabels", + "schema": "schema_test", + "table": "adcreatives_adlabels" + }, + "schema_test.adcreatives_image_crops.image_crops": { + "file": "adcreatives_image_crops", + "schema": "schema_test", + "table": "adcreatives_image_crops" + }, + "schema_test.adcreatives_image_crops_100x100.100x100": { + "file": "adcreatives_image_crops_100x100", + "schema": "schema_test", + "table": "adcreatives_image_crops_100x100" + }, + "schema_test.adcreatives_image_crops_100x72.100x72": { + "file": "adcreatives_image_crops_100x72", + "schema": "schema_test", + "table": "adcreatives_image_crops_100x72" + }, + "schema_test.adcreatives_image_crops_191x100.191x100": { + "file": "adcreatives_image_crops_191x100", + "schema": "schema_test", + "table": "adcreatives_image_crops_191x100" + }, + "schema_test.adcreatives_image_crops_400x150.400x150": { + "file": "adcreatives_image_crops_400x150", + "schema": "schema_test", + "table": "adcreatives_image_crops_400x150" + }, + "schema_test.adcreatives_image_crops_400x500.400x500": { + "file": "adcreatives_image_crops_400x500", + "schema": "schema_test", + "table": "adcreatives_image_crops_400x500" + }, + "schema_test.adcreatives_image_crops_600x360.600x360": { + "file": "adcreatives_image_crops_600x360", + "schema": "schema_test", + "table": "adcreatives_image_crops_600x360" + }, + "schema_test.adcreatives_image_crops_90x160.90x160": { + "file": "adcreatives_image_crops_90x160", + "schema": "schema_test", + "table": "adcreatives_image_crops_90x160" + }, + "schema_test.adcreatives_object_story_spec.object_story_spec": { + "file": "adcreatives_object_story_spec", + "schema": "schema_test", + "table": "adcreatives_object_story_spec" + }, + "schema_test.adcreatives_object_story_spec_link_data.link_data": { + "file": "adcreatives_object_story_spec_link_data", + "schema": "schema_test", + "table": "adcreatives_object_story_spec_link_data" + }, + "schema_test.adcreatives_object_story_spec_link_data_app_link_spec.app_link_spec": { + "file": "adcreatives_object_st__nk_data_app_link_spec", + "schema": "schema_test", + "table": "adcreatives_object_st__nk_data_app_link_spec" + }, + "schema_test.adcreatives_object_story_spec_link_data_app_link_spec_android.android": { + "file": "adcreatives_object_st__app_link_spec_android", + "schema": "schema_test", + "table": "adcreatives_object_st__app_link_spec_android" + }, + "schema_test.adcreatives_object_story_spec_link_data_app_link_spec_ios.ios": { + "file": "adcreatives_object_st__ata_app_link_spec_ios", + "schema": "schema_test", + "table": "adcreatives_object_st__ata_app_link_spec_ios" + }, + "schema_test.adcreatives_object_story_spec_link_data_app_link_spec_ipad.ipad": { + "file": "adcreatives_object_st__ta_app_link_spec_ipad", + "schema": "schema_test", + "table": "adcreatives_object_st__ta_app_link_spec_ipad" + }, + "schema_test.adcreatives_object_story_spec_link_data_app_link_spec_iphone.iphone": { + "file": "adcreatives_object_st___app_link_spec_iphone", + "schema": "schema_test", + "table": "adcreatives_object_st___app_link_spec_iphone" + }, + "schema_test.adcreatives_object_story_spec_link_data_image_crops.image_crops": { + "file": "adcreatives_object_st__link_data_image_crops", + "schema": "schema_test", + "table": "adcreatives_object_st__link_data_image_crops" + }, + "schema_test.adcreatives_object_story_spec_link_data_image_crops_100x100.100x100": { + "file": "adcreatives_object_st__a_image_crops_100x100", + "schema": "schema_test", + "table": "adcreatives_object_st__a_image_crops_100x100" + }, + "schema_test.adcreatives_object_story_spec_link_data_image_crops_100x72.100x72": { + "file": "adcreatives_object_st__ta_image_crops_100x72", + "schema": "schema_test", + "table": "adcreatives_object_st__ta_image_crops_100x72" + }, + "schema_test.adcreatives_object_story_spec_link_data_image_crops_191x100.191x100": { + "file": "adcreatives_object_st__a_image_crops_191x100", + "schema": "schema_test", + "table": "adcreatives_object_st__a_image_crops_191x100" + }, + "schema_test.adcreatives_object_story_spec_link_data_image_crops_400x150.400x150": { + "file": "adcreatives_object_st__a_image_crops_400x150", + "schema": "schema_test", + "table": "adcreatives_object_st__a_image_crops_400x150" + }, + "schema_test.adcreatives_object_story_spec_link_data_image_crops_400x500.400x500": { + "file": "adcreatives_object_st__a_image_crops_400x500", + "schema": "schema_test", + "table": "adcreatives_object_st__a_image_crops_400x500" + }, + "schema_test.adcreatives_object_story_spec_link_data_image_crops_600x360.600x360": { + "file": "adcreatives_object_st__a_image_crops_600x360", + "schema": "schema_test", + "table": "adcreatives_object_st__a_image_crops_600x360" + }, + "schema_test.adcreatives_object_story_spec_link_data_image_crops_90x160.90x160": { + "file": "adcreatives_object_st__ta_image_crops_90x160", + "schema": "schema_test", + "table": "adcreatives_object_st__ta_image_crops_90x160" + } +} diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/resources/un-nesting_collisions_catalog_expected_mssql_names.json b/airbyte-integrations/bases/base-normalization/unit_tests/resources/un-nesting_collisions_catalog_expected_mssql_names.json new file mode 100644 index 000000000000..0ae55a762fd8 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/unit_tests/resources/un-nesting_collisions_catalog_expected_mssql_names.json @@ -0,0 +1,52 @@ +{ + "_airbyte_namespace.simple stream name.simple stream name": { + "file": "simple_stream_name_f35", + "schema": "_airbyte_namespace", + "table": "simple_stream_name_f35" + }, + "namespace.simple stream name.simple stream name": { + "file": "simple_stream_name_f35", + "schema": "namespace", + "table": "simple_stream_name_f35" + }, + "_airbyte_namespace.simple_stream_name.stream_name": { + "file": "_airbyte_namespace_simple_b94_stream_name", + "schema": "_airbyte_namespace", + "table": "simple_b94_stream_name" + }, + "namespace.simple_stream_name.stream_name": { + "file": "namespace_simple_b94_stream_name", + "schema": "namespace", + "table": "simple_b94_stream_name" + }, + "_airbyte_namespace.simple.simple": { + "file": "simple", + "schema": "_airbyte_namespace", + "table": "simple" + }, + "namespace.simple.simple": { + "file": "simple", + "schema": "namespace", + "table": "simple" + }, + "_airbyte_other_namespace.simple_b94_stream_name.simple_b94_stream_name": { + "file": "_airbyte_other_namesp__e_b94_stream_name_f9d", + "schema": "_airbyte_other_namespace", + "table": "simple_b94_stream_name" + }, + "other_namespace.simple_b94_stream_name.simple_b94_stream_name": { + "file": "other_namespace_simple_b94_stream_name", + "schema": "other_namespace", + "table": "simple_b94_stream_name" + }, + "_airbyte_yet_another_namespace_with_a_very_long_name.simple_b94_stream_name.simple_b94_stream_name": { + "file": "_airbyte_yet_another___e_b94_stream_name_bae", + "schema": "_airbyte_yet_another_namespace_with_a_very_long_name", + "table": "simple_b94_stream_name" + }, + "yet_another_namespace_with_a_very_long_name.simple_b94_stream_name.simple_b94_stream_name": { + "file": "yet_another_namespace__e_b94_stream_name_5d1", + "schema": "yet_another_namespace_with_a_very_long_name", + "table": "simple_b94_stream_name" + } +} diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_destination_name_transformer.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_destination_name_transformer.py index 3c9ef14796e0..e5c4a0d2eccd 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/test_destination_name_transformer.py +++ b/airbyte-integrations/bases/base-normalization/unit_tests/test_destination_name_transformer.py @@ -36,24 +36,28 @@ def before_tests(request): ("Hello World", "Snowflake", True), ("Hello World", "Redshift", True), ("Hello World", "MySQL", True), + ("Hello World", "MSSQL", True), # Reserved Word for BigQuery and MySQL only ("Groups", "Postgres", False), ("Groups", "BigQuery", True), ("Groups", "Snowflake", False), ("Groups", "Redshift", False), ("Groups", "MySQL", True), + ("Groups", "MSSQL", False), # Doesnt start with alpha or underscore ("100x200", "Postgres", True), ("100x200", "BigQuery", False), ("100x200", "Snowflake", True), ("100x200", "Redshift", True), ("100x200", "MySQL", True), + ("100x200", "MSSQL", True), # Contains non alpha numeric ("post.wall", "Postgres", True), ("post.wall", "BigQuery", False), ("post.wall", "Snowflake", True), ("post.wall", "Redshift", True), ("post.wall", "MySQL", True), + ("post.wall", "MSSQL", True), ], ) def test_needs_quote(input_str: str, destination_type: str, expected: bool): @@ -103,48 +107,56 @@ def test_transform_standard_naming(input_str: str, expected: str): ("Identifier Name3", "Snowflake", "IDENTIFIER_NAME3", "{{ adapter.quote('Identifier Name3') }}"), ("Identifier Name4", "Redshift", "identifier_name4", "{{ adapter.quote('identifier name4') }}"), ("Identifier Name5", "MySQL", "identifier_name5", "{{ adapter.quote('Identifier Name5') }}"), + ("Identifier Name6", "MSSQL", "identifier_name6", "{{ adapter.quote('Identifier Name6') }}"), # Unicode ("a-Unicode_name_文1", "Postgres", "a_unicode_name__1", "{{ adapter.quote('a-Unicode_name_文1') }}"), ("a-Unicode_name_文2", "BigQuery", "a_Unicode_name__2", "a_Unicode_name__2"), ("a-Unicode_name_文3", "Snowflake", "A_UNICODE_NAME__3", "{{ adapter.quote('a-Unicode_name_文3') }}"), ("a-Unicode_name_文4", "Redshift", "a_unicode_name__4", "{{ adapter.quote('a-unicode_name_文4') }}"), ("a-Unicode_name_文5", "MySQL", "a_unicode_name__5", "{{ adapter.quote('a-Unicode_name_文5') }}"), + ("a-Unicode_name_文6", "MSSQL", "a_unicode_name__6", "{{ adapter.quote('a-Unicode_name_文6') }}"), # Doesnt start with alpha or underscore ("100x2001", "Postgres", "100x2001", "{{ adapter.quote('100x2001') }}"), ("100x2002", "BigQuery", "_100x2002", "_100x2002"), ("100x2003", "Snowflake", "100x2003", "{{ adapter.quote('100x2003') }}"), ("100x2004", "Redshift", "100x2004", "{{ adapter.quote('100x2004') }}"), ("100x2005", "MySQL", "100x2005", "{{ adapter.quote('100x2005') }}"), + ("100x2006", "MSSQL", "100x2006", "{{ adapter.quote('100x2006') }}"), # Reserved Keywords in BQ and MySQL ("Groups", "Postgres", "groups", "groups"), ("Groups", "BigQuery", "Groups", "{{ adapter.quote('Groups') }}"), ("Groups", "Snowflake", "GROUPS", "GROUPS"), ("Groups", "Redshift", "groups", "groups"), ("Groups", "MySQL", "Groups", "{{ adapter.quote('Groups') }}"), + ("Groups", "MSSQL", "groups", "groups"), # Reserved Keywords ("DisTincT", "Postgres", "DisTincT", "{{ adapter.quote('DisTincT') }}"), ("DisTincT", "BigQuery", "DisTincT", "{{ adapter.quote('DisTincT') }}"), ("DisTincT", "Snowflake", "DisTincT", "{{ adapter.quote('DisTincT') }}"), ("DisTincT", "Redshift", "distinct", "{{ adapter.quote('distinct') }}"), ("DisTincT", "MySQL", "DisTincT", "{{ adapter.quote('DisTincT') }}"), + ("DisTincT", "MSSQL", "DisTincT", "{{ adapter.quote('DisTincT') }}"), # Quoted identifiers ("'QuoTed1 IdenTifiER'", "Postgres", "_quoted1_identifier_", "{{ adapter.quote('\\'QuoTed1 IdenTifiER\\'') }}"), ("'QuoTed2 IdenTifiER'", "BigQuery", "_QuoTed2_IdenTifiER_", "_QuoTed2_IdenTifiER_"), ("'QuoTed3 IdenTifiER'", "Snowflake", "_QUOTED3_IDENTIFIER_", "{{ adapter.quote('\\'QuoTed3 IdenTifiER\\'') }}"), ("'QuoTed4 IdenTifiER'", "Redshift", "_quoted4_identifier_", "{{ adapter.quote('\\'quoted4 identifier\\'') }}"), ("'QuoTed5 IdenTifiER'", "MySQL", "_quoted5_identifier_", "{{ adapter.quote('\\'QuoTed5 IdenTifiER\\'') }}"), + ("'QuoTed6 IdenTifiER'", "MSSQL", "_quoted6_identifier_", "{{ adapter.quote('\\'QuoTed6 IdenTifiER\\'') }}"), # Double Quoted identifiers - ('"QuoTed6 IdenTifiER"', "Postgres", "_quoted6_identifier_", '{{ adapter.quote(\'""QuoTed6 IdenTifiER""\') }}'), - ('"QuoTed7 IdenTifiER"', "BigQuery", "_QuoTed7_IdenTifiER_", "_QuoTed7_IdenTifiER_"), - ('"QuoTed8 IdenTifiER"', "Snowflake", "_QUOTED8_IDENTIFIER_", '{{ adapter.quote(\'""QuoTed8 IdenTifiER""\') }}'), - ('"QuoTed9 IdenTifiER"', "Redshift", "_quoted9_identifier_", '{{ adapter.quote(\'""quoted9 identifier""\') }}'), - ('"QuoTed10 IdenTifiER"', "MySQL", "_quoted10_identifier_", "{{ adapter.quote('\"QuoTed10 IdenTifiER\"') }}"), + ('"QuoTed7 IdenTifiER"', "Postgres", "_quoted7_identifier_", '{{ adapter.quote(\'""QuoTed7 IdenTifiER""\') }}'), + ('"QuoTed8 IdenTifiER"', "BigQuery", "_QuoTed8_IdenTifiER_", "_QuoTed8_IdenTifiER_"), + ('"QuoTed9 IdenTifiER"', "Snowflake", "_QUOTED9_IDENTIFIER_", '{{ adapter.quote(\'""QuoTed9 IdenTifiER""\') }}'), + ('"QuoTed10 IdenTifiER"', "Redshift", "_quoted10_identifier_", '{{ adapter.quote(\'""quoted10 identifier""\') }}'), + ('"QuoTed11 IdenTifiER"', "MySQL", "_quoted11_identifier_", "{{ adapter.quote('\"QuoTed11 IdenTifiER\"') }}"), + ('"QuoTed12 IdenTifiER"', "MSSQL", "_quoted12_identifier_", '{{ adapter.quote(\'""QuoTed12 IdenTifiER""\') }}'), # Back Quoted identifiers - ("`QuoTed11 IdenTifiER`", "Postgres", "_quoted11_identifier_", "{{ adapter.quote('`QuoTed11 IdenTifiER`') }}"), - ("`QuoTed12 IdenTifiER`", "BigQuery", "_QuoTed12_IdenTifiER_", "_QuoTed12_IdenTifiER_"), - ("`QuoTed13 IdenTifiER`", "Snowflake", "_QUOTED13_IDENTIFIER_", "{{ adapter.quote('`QuoTed13 IdenTifiER`') }}"), - ("`QuoTed14 IdenTifiER`", "Redshift", "_quoted14_identifier_", "{{ adapter.quote('`quoted14 identifier`') }}"), - ("`QuoTed15 IdenTifiER`", "MySQL", "_quoted15_identifier_", "{{ adapter.quote('_QuoTed15 IdenTifiER_') }}"), + ("`QuoTed13 IdenTifiER`", "Postgres", "_quoted13_identifier_", "{{ adapter.quote('`QuoTed13 IdenTifiER`') }}"), + ("`QuoTed14 IdenTifiER`", "BigQuery", "_QuoTed14_IdenTifiER_", "_QuoTed14_IdenTifiER_"), + ("`QuoTed15 IdenTifiER`", "Snowflake", "_QUOTED15_IDENTIFIER_", "{{ adapter.quote('`QuoTed15 IdenTifiER`') }}"), + ("`QuoTed16 IdenTifiER`", "Redshift", "_quoted16_identifier_", "{{ adapter.quote('`quoted16 identifier`') }}"), + ("`QuoTed17 IdenTifiER`", "MySQL", "_quoted17_identifier_", "{{ adapter.quote('_QuoTed17 IdenTifiER_') }}"), + ("`QuoTed18 IdenTifiER`", "MSSQL", "_quoted18_identifier_", "{{ adapter.quote('`QuoTed18 IdenTifiER`') }}"), ], ) def test_normalize_name(input_str: str, destination_type: str, expected: str, expected_column: str): @@ -163,12 +175,14 @@ def test_normalize_name(input_str: str, destination_type: str, expected: str, ex ("Identifier Name", "Snowflake", "{{ adapter.quote('Identifier Name') }}", "adapter.quote('Identifier Name')"), ("Identifier Name", "Redshift", "{{ adapter.quote('identifier name') }}", "adapter.quote('identifier name')"), ("Identifier Name", "MySQL", "{{ adapter.quote('Identifier Name') }}", "adapter.quote('Identifier Name')"), + ("Identifier Name", "MSSQL", "{{ adapter.quote('Identifier Name') }}", "adapter.quote('Identifier Name')"), # Reserved Word for BigQuery and MySQL only ("Groups", "Postgres", "groups", "'groups'"), ("Groups", "BigQuery", "{{ adapter.quote('Groups') }}", "adapter.quote('Groups')"), ("Groups", "Snowflake", "GROUPS", "'GROUPS'"), ("Groups", "Redshift", "groups", "'groups'"), ("Groups", "MySQL", "{{ adapter.quote('Groups') }}", "adapter.quote('Groups')"), + ("Groups", "MSSQL", "groups", "'groups'"), ], ) def test_normalize_column_name(input_str: str, destination_type: str, expected: str, expected_in_jinja: str): diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py index 3d70a0777e3c..1adc28ba4cae 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py +++ b/airbyte-integrations/bases/base-normalization/unit_tests/test_transform_config.py @@ -284,6 +284,32 @@ def test_transform_mysql(self): # DBT schema is equivalent to MySQL database assert extract_schema(actual) == "my_db" + def test_transform_mssql(self): + input = { + "type": "sqlserver", + "host": "airbyte.io", + "port": 1433, + "database": "my_db", + "schema": "my_db", + "username": "SA", + "password": "password1234", + } + + actual = TransformConfig().transform_mysql(input) + expected = { + "type": "sqlserver", + "server": "airbyte.io", + "port": 1433, + "schema": "my_db", + "database": "my_db", + "username": "SA", + "password": "password1234", + } + + assert expected == actual + # DBT schema is equivalent to MySQL database + assert extract_schema(actual) == "my_db" + # test that the full config is produced. this overlaps slightly with the transform_postgres test. def test_transform(self): input = { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java index 8e560ad2dedb..fb0d78f8d938 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java @@ -28,7 +28,7 @@ public class DefaultNormalizationRunner implements NormalizationRunner { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultNormalizationRunner.class); - public static final String NORMALIZATION_IMAGE_NAME = "airbyte/normalization:0.1.49"; + public static final String NORMALIZATION_IMAGE_NAME = "airbyte/normalization:0.1.50"; private final DestinationType destinationType; private final ProcessFactory processFactory; @@ -40,7 +40,8 @@ public enum DestinationType { POSTGRES, REDSHIFT, SNOWFLAKE, - MYSQL + MYSQL, + MSSQL } public DefaultNormalizationRunner(final DestinationType destinationType, final ProcessFactory processFactory) { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java index 6a9bd824ca3e..c21f55b30149 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java @@ -18,6 +18,7 @@ public class NormalizationRunnerFactory { .put("airbyte/destination-redshift", DefaultNormalizationRunner.DestinationType.REDSHIFT) .put("airbyte/destination-snowflake", DefaultNormalizationRunner.DestinationType.SNOWFLAKE) .put("airbyte/destination-mysql", DefaultNormalizationRunner.DestinationType.MYSQL) + .put("airbyte/destination-mssql", DefaultNormalizationRunner.DestinationType.MSSQL) .build(); public static NormalizationRunner create(final String imageName, final ProcessFactory processFactory) { diff --git a/build.gradle b/build.gradle index a899fcace8f2..05de2ae337be 100644 --- a/build.gradle +++ b/build.gradle @@ -68,6 +68,8 @@ def createSpotlessTarget = { pattern -> 'dbt-project-template', 'dbt_data_tests', 'dbt_schema_tests', + 'dbt_data_tests_mssql', + 'dbt_tmp_data_test_mssql', 'normalization_test_output', 'tools', 'secrets', diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index ab7f18610130..410cb62752b6 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -28,12 +28,28 @@ Each stream will be output into its own table in SQL Server. Each table will con ## Getting Started (Airbyte Cloud) Airbyte Cloud only supports connecting to your MSSQL instance with TLS encryption. Other than that, you can proceed with the open-source instructions below. +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Deduped History | Yes | | +| Namespaces | Yes | | + ## Getting Started (Airbyte Open-Source) ### Requirements To use the SQL Server destination, you'll need: +MS SQL Server: `Azure SQL Database`, `Azure Synapse Analytics`, `Azure SQL Managed Instance`, `SQL Server 2019`, `SQL Server 2017`, `SQL Server 2016`, `SQL Server 2014`, `SQL Server 2012`, or `PDW 2008R2 AU34`. + +### Normalization Requirements + +To sync **with** normalization you'll need to use MS SQL Server of the following versions: +`SQL Server 2019`, `SQL Server 2017`, `SQL Server 2016`, `SQL Server 2014`. +The work of normalization on `SQL Server 2012` and bellow are not guaranteed. + +### Setup guide * MS SQL Server: `Azure SQL Database`, `Azure Synapse Analytics`, `Azure SQL Managed Instance`, `SQL Server 2019`, `SQL Server 2017`, `SQL Server 2016`, `SQL Server 2014`, `SQL Server 2012`, or `PDW 2008R2 AU34`. #### Network Access diff --git a/settings.gradle b/settings.gradle index 0507e111236b..38601672cdce 100644 --- a/settings.gradle +++ b/settings.gradle @@ -88,6 +88,7 @@ if(!System.getenv().containsKey("SUB_BUILD") || System.getenv().get("SUB_BUILD") include ':airbyte-integrations:connectors:destination-redshift' include ':airbyte-integrations:connectors:destination-snowflake' include ':airbyte-integrations:connectors:destination-oracle' + include ':airbyte-integrations:connectors:destination-mssql' //Needed by destination-bugquery include ':airbyte-integrations:connectors:destination-s3'