From 712b4a83d9fc58cf212ed48da1c8a7e79d1085fb Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Tue, 6 Dec 2022 16:54:25 +0200 Subject: [PATCH 1/4] Enable full SAT for the snowflake --- .../acceptance-test-config.yml | 16 +++++++ .../integration_tests/configured_catalog.json | 19 ++++++++ .../configured_catalog_inc.json | 16 +++++++ .../integration_tests/expected_records.json | 7 +++ .../integration_tests/sat_basic_dataset.sql | 45 ++++++++++++++++++ .../integration_tests/sat_full_dataset.sql | 46 +++++++++++++++++++ 6 files changed, 149 insertions(+) create mode 100644 airbyte-integrations/connectors/source-snowflake/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-snowflake/integration_tests/configured_catalog_inc.json create mode 100644 airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.json create mode 100644 airbyte-integrations/connectors/source-snowflake/integration_tests/sat_basic_dataset.sql create mode 100644 airbyte-integrations/connectors/source-snowflake/integration_tests/sat_full_dataset.sql diff --git a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml index d359de152b5e..197fccc0591d 100644 --- a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml @@ -5,3 +5,19 @@ tests: spec: - spec_path: "src/test-integration/resources/expected_spec.json" config_path: "secrets/config.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + expect_records: + path: "integration_tests/expected_records.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_inc.json" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-snowflake/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..2bcf98da2bcf --- /dev/null +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/configured_catalog.json @@ -0,0 +1,19 @@ +{ + "streams": [ + { + "stream": { + "name": "SAT_BASIC_DATASET", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["ID"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/configured_catalog_inc.json b/airbyte-integrations/connectors/source-snowflake/integration_tests/configured_catalog_inc.json new file mode 100644 index 000000000000..52816ea6df67 --- /dev/null +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/configured_catalog_inc.json @@ -0,0 +1,16 @@ +{ + "streams": [ + { + "stream": { + "name": "SAT_BASIC_DATASET", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["ID"] + }, + "sync_mode": "incremental", + "cursor_field": ["ID"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.json b/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.json new file mode 100644 index 000000000000..bb63fbe99f5a --- /dev/null +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.json @@ -0,0 +1,7 @@ +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 1, "TEST_COLUMN_1" : 99999999999999999999999999999999999999, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : -9007199254740990.0, "TEST_COLUMN_12" : 1e-307, "TEST_COLUMN_14" : "\u0442\u0435\u0441\u0442", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-!-", "TEST_COLUMN_17" : "a", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : 9223372036854775807, "TEST_COLUMN_20" : "0001-01-01", "TEST_COLUMN_21" : "0001-01-01T00:00:00.000000", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123000", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123000Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123000", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123000Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : 99999999999999999999999999999999999999, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n -122.35,\n 37.55\n ],\n \"type\": \"Point\"\n}", "TEST_COLUMN_4" : 99999999999999999999999999999999999999, "TEST_COLUMN_5" : 9223372036854775807, "TEST_COLUMN_6" : 9223372036854775807, "TEST_COLUMN_7" : 9223372036854775807, "TEST_COLUMN_8" : 9223372036854775807, "TEST_COLUMN_9" : 9223372036854775807 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 2, "TEST_COLUMN_1" : -99999999999999999999999999999999999999, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "\u26a1 test \ufffd\ufffd", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u30b9", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.000000", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -99999999999999999999999999999999999999, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 3, "TEST_COLUMN_1" : 9223372036854775807, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "!", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : 9223372036854775807, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 4, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 5, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 6, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 7, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_basic_dataset.sql b/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_basic_dataset.sql new file mode 100644 index 000000000000..0dcbc56151c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_basic_dataset.sql @@ -0,0 +1,45 @@ +create schema sat_test_dataset; + +-- Uncomment the line below if you need to recreate the table. +-- DROP TABLE sat_test_dataset.sat_basic_dataset; + +CREATE TABLE sat_test_dataset.sat_basic_dataset +( + ID INTEGER PRIMARY KEY, + TEST_COLUMN_1 NUMBER, + TEST_COLUMN_10 NUMBER(10,5), + TEST_COLUMN_11 DOUBLE, + TEST_COLUMN_12 FLOAT, + TEST_COLUMN_14 VARCHAR, + TEST_COLUMN_15 STRING, + TEST_COLUMN_16 TEXT, + TEST_COLUMN_17 CHAR, + TEST_COLUMN_18 BINARY, + TEST_COLUMN_19 BOOLEAN, + TEST_COLUMN_2 DECIMAL, + TEST_COLUMN_20 DATE, + TEST_COLUMN_21 DATETIME, + TEST_COLUMN_23 TIMESTAMP, + TEST_COLUMN_24 TIMESTAMP_LTZ, + TEST_COLUMN_25 TIMESTAMP_NTZ, + TEST_COLUMN_26 TIMESTAMP_TZ, + TEST_COLUMN_27 VARIANT, + TEST_COLUMN_28 ARRAY, + TEST_COLUMN_29 OBJECT, + TEST_COLUMN_3 NUMERIC, + TEST_COLUMN_30 GEOGRAPHY, + TEST_COLUMN_4 BIGINT, + TEST_COLUMN_5 INT, + TEST_COLUMN_6 BIGINT, + TEST_COLUMN_7 SMALLINT, + TEST_COLUMN_8 TINYINT, + TEST_COLUMN_9 BYTEINT +); + +INSERT INTO sat_test_dataset.sat_basic_dataset select 1, 99999999999999999999999999999999999999, 10.12345, -9007199254740991, 10e-308, 'тест', 'テスト', '-!-', 'a', to_binary('HELP', 'UTF-8'), 'true', 9223372036854775807, '0001-01-01', '0001-01-01 00:00:00', '2018-03-22 12:00:00.123', '2018-03-22 12:00:00.123 +05:00', '2018-03-22 12:00:00.123 +05:00', '2018-03-22 12:00:00.123 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), 99999999999999999999999999999999999999, 'POINT(-122.35 37.55)', 99999999999999999999999999999999999999, 9223372036854775807, 9223372036854775807, 9223372036854775807, 9223372036854775807, 9223372036854775807; +INSERT INTO sat_test_dataset.sat_basic_dataset select 2, -99999999999999999999999999999999999999, 10.12345, 9007199254740991, 10e+307, '⚡ test ��', 'テスト', '-\x25-', 'ス', to_binary('HELP', 'UTF-8'), 5, -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -99999999999999999999999999999999999999, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT INTO sat_test_dataset.sat_basic_dataset select 3, 9223372036854775807, 10.12345, 9007199254740991, 10e+307, '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', '!', to_binary('HELP', 'UTF-8'), 'false', -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), 9223372036854775807, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT INTO sat_test_dataset.sat_basic_dataset select 4, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), 0, -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT INTO sat_test_dataset.sat_basic_dataset select 5, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('y'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT INTO sat_test_dataset.sat_basic_dataset select 6, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('n'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT INTO sat_test_dataset.sat_basic_dataset select 7, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('n'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_full_dataset.sql b/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_full_dataset.sql new file mode 100644 index 000000000000..3726271d9d98 --- /dev/null +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/sat_full_dataset.sql @@ -0,0 +1,46 @@ +create schema sat_test_dataset; + +-- Uncomment the line below if you need to recreate the table. +-- DROP TABLE sat_test_dataset.sat_full_dataset; + +CREATE TABLE sat_test_dataset.sat_full_dataset +( + ID INTEGER PRIMARY KEY, + TEST_COLUMN_1 NUMBER, + TEST_COLUMN_10 NUMBER(10,5), + TEST_COLUMN_11 DOUBLE, + TEST_COLUMN_12 FLOAT, + TEST_COLUMN_14 VARCHAR, + TEST_COLUMN_15 STRING, + TEST_COLUMN_16 TEXT, + TEST_COLUMN_17 CHAR, + TEST_COLUMN_18 BINARY, + TEST_COLUMN_19 BOOLEAN, + TEST_COLUMN_2 DECIMAL, + TEST_COLUMN_20 DATE, + TEST_COLUMN_21 DATETIME, + TEST_COLUMN_22 TIME, + TEST_COLUMN_23 TIMESTAMP, + TEST_COLUMN_24 TIMESTAMP_LTZ, + TEST_COLUMN_25 TIMESTAMP_NTZ, + TEST_COLUMN_26 TIMESTAMP_TZ, + TEST_COLUMN_27 VARIANT, + TEST_COLUMN_28 ARRAY, + TEST_COLUMN_29 OBJECT, + TEST_COLUMN_3 NUMERIC, + TEST_COLUMN_30 GEOGRAPHY, + TEST_COLUMN_4 BIGINT, + TEST_COLUMN_5 INT, + TEST_COLUMN_6 BIGINT, + TEST_COLUMN_7 SMALLINT, + TEST_COLUMN_8 TINYINT, + TEST_COLUMN_9 BYTEINT +); + +INSERT INTO sat_test_dataset.sat_full_dataset select 1, 99999999999999999999999999999999999999, 10.12345, -9007199254740991, 10e-308, 'тест', 'テスト', '-!-', 'a', to_binary('HELP', 'UTF-8'), 'true', 9223372036854775807, '0001-01-01', '0001-01-01 00:00:00', '00:00:00', '2018-03-22 12:00:00.123', '2018-03-22 12:00:00.123 +05:00', '2018-03-22 12:00:00.123 +05:00', '2018-03-22 12:00:00.123 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), 99999999999999999999999999999999999999, 'POINT(-122.35 37.55)', 99999999999999999999999999999999999999, 9223372036854775807, 9223372036854775807, 9223372036854775807, 9223372036854775807, 9223372036854775807; +INSERT INTO sat_test_dataset.sat_full_dataset select 2, -99999999999999999999999999999999999999, 10.12345, 9007199254740991, 10e+307, '⚡ test ��', 'テスト', '-\x25-', 'ス', to_binary('HELP', 'UTF-8'), 5, -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59', '1:59 PM', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -99999999999999999999999999999999999999, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT INTO sat_test_dataset.sat_full_dataset select 3, 9223372036854775807, 10.12345, 9007199254740991, 10e+307, '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', '!', to_binary('HELP', 'UTF-8'), 'false', -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), 9223372036854775807, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT INTO sat_test_dataset.sat_full_dataset select 4, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), 0, -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT INTO sat_test_dataset.sat_full_dataset select 5, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('y'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT INTO sat_test_dataset.sat_full_dataset select 6, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('n'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; +INSERT INTO sat_test_dataset.sat_full_dataset select 7, -9223372036854775808, 10.12345, 9007199254740991, 10e+307, '!"#$%&\'()*+,-./:;<=>?\@[\]^_\`{|}~', 'テスト', '-\x25-', 'ї', to_binary('HELP', 'UTF-8'), TO_BOOLEAN('n'), -9223372036854775808, '9999-12-31', '9999-12-31 23:59:59.123456', '23:59:59.123456', '2018-03-22 12:00:00.123456', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', '2018-03-22 12:00:00.123456 +05:00', parse_json(' { "key1": "value1", "key2": "value2" } '), array_construct(1, 2, 3), parse_json(' { "outer_key1": { "inner_key1A": "1a", "inner_key1B": "1b" }, "outer_key2": { "inner_key2": 2 } } '), -9223372036854775808, 'LINESTRING(-124.20 42.00, -120.01 41.99)', -99999999999999999999999999999999999999, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808, -9223372036854775808; \ No newline at end of file From 8ecc9a9f4f74576f70a107217c3c522940e53557 Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 7 Dec 2022 16:40:26 +0200 Subject: [PATCH 2/4] readme + format --- .../acceptance-test-config.yml | 34 +++++++++++-------- .../integration_tests/README.md | 3 ++ 2 files changed, 23 insertions(+), 14 deletions(-) create mode 100644 airbyte-integrations/connectors/source-snowflake/integration_tests/README.md diff --git a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml index 197fccc0591d..5547d866ac94 100644 --- a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml @@ -1,23 +1,29 @@ # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-snowflake:dev -tests: +acceptance_tests: spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "secrets/config.json" + tests: + - spec_path: "src/test-integration/resources/expected_spec.json" + config_path: "secrets/config.json" connection: - - config_path: "secrets/config.json" - status: "succeed" + tests: + - config_path: "secrets/config.json" + status: "succeed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - expect_records: - path: "integration_tests/expected_records.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + expect_records: + path: "integration_tests/expected_records.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_inc.json" \ No newline at end of file + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_inc.json" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/README.md b/airbyte-integrations/connectors/source-snowflake/integration_tests/README.md new file mode 100644 index 000000000000..96aa5492669b --- /dev/null +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/README.md @@ -0,0 +1,3 @@ +# Seeding the dataset +You can find the SQL scripts in this folder if you need to create or fix the SAT dataset. +For more instructions and information about valid scripts, please check this [doc](https://docs.google.com/document/d/1k5TvxaNhKdr44aJIHWWtLk14Tzd2gbNX-J8YNoTj8u0/edit#heading=h.ls9oiedt9wyy). From 9a28182b80c120a12e4e096b2676788b6e21d386 Mon Sep 17 00:00:00 2001 From: Andrii Leonets <30464745+DoNotPanicUA@users.noreply.github.com> Date: Wed, 7 Dec 2022 17:00:00 +0200 Subject: [PATCH 3/4] Update airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml Co-authored-by: Augustin --- .../connectors/source-snowflake/acceptance-test-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml index 5547d866ac94..f1b19fa8dd34 100644 --- a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml @@ -1,6 +1,7 @@ # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-snowflake:dev +test_strictness_level: high acceptance_tests: spec: tests: From 88cccb61838f8de90d7d3f7babfd6de98897b40a Mon Sep 17 00:00:00 2001 From: "andrii.leonets" Date: Wed, 7 Dec 2022 17:29:40 +0200 Subject: [PATCH 4/4] Revert "Update airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml" This reverts commit 9a28182b80c120a12e4e096b2676788b6e21d386. --- .../connectors/source-snowflake/acceptance-test-config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml index f1b19fa8dd34..5547d866ac94 100644 --- a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml @@ -1,7 +1,6 @@ # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-snowflake:dev -test_strictness_level: high acceptance_tests: spec: tests: