Enable full SAT for the Redshift source (#19915)

* enable SAT for redshift * add bigger timeouts + specify catalogs (default works wrong) * new line * readme * new format Co-authored-by: Augustin <augustin@airbyte.io>
airbytehq · Dec 8, 2022 · c2e4c3e · c2e4c3e
1 parent dc71194
commit c2e4c3e
Show file tree

Hide file tree

Showing 6 changed files with 91 additions and 3 deletions.
diff --git a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml b/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml
@@ -1,7 +1,35 @@
 # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
 # for more information about how to configure these tests
 connector_image: airbyte/source-redshift:dev
-tests:
+acceptance_tests:
   spec:
-    - spec_path: "src/test-integration/resources/expected_spec.json"
-      config_path: "secrets/config.json"
+    tests:
+      - spec_path: "src/test-integration/resources/expected_spec.json"
+        timeout_seconds: "1200"
+        config_path: "secrets/config.json"
+  connection:
+    tests:
+      - config_path: "secrets/config.json"
+        timeout_seconds: "1200"
+        status: "succeed"
+  discovery:
+    tests:
+      - config_path: "secrets/config.json"
+        timeout_seconds: "1200"
+  basic_read:
+    tests:
+      - config_path: "secrets/config.json"
+        timeout_seconds: "1200"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        expect_records:
+          path: "integration_tests/expected_records.json"
+  full_refresh:
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        timeout_seconds: "1200"
+  incremental:
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog_inc.json"
+        timeout_seconds: "1200"
diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/README.md b/airbyte-integrations/connectors/source-redshift/integration_tests/README.md
@@ -0,0 +1,3 @@
+# Seeding the dataset
+You can find the SQL scripts in this folder if you need to create or fix the SAT dataset.
+For more instructions and information about valid scripts, please check this [doc](https://docs.google.com/document/d/1k5TvxaNhKdr44aJIHWWtLk14Tzd2gbNX-J8YNoTj8u0/edit#heading=h.ls9oiedt9wyy).
diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql b/airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql
@@ -0,0 +1,20 @@
+CREATE TABLE IF NOT EXISTS sat_test_dataset.sat_basic_dataset (
+                                                                  id INTEGER,
+                                                                  test_column_1 SMALLINT,
+                                                                  test_column_2 INTEGER,
+                                                                  test_column_3 BIGINT,
+                                                                  test_column_4 DECIMAL,
+                                                                  test_column_5 REAL,
+                                                                  test_column_6 DOUBLE PRECISION,
+                                                                  test_column_7 BOOLEAN,
+                                                                  test_column_8 CHAR,
+                                                                  test_column_9 VARCHAR,
+                                                                  test_column_10 DATE,
+                                                                  test_column_11 TIMESTAMP,
+                                                                  test_column_12 TIMESTAMPTZ,
+                                                                  test_column_13 TIME,
+                                                                  test_column_14 TIMETZ,
+                                                                  test_column_15 VARBYTE);
+
+insert into sat_test_dataset.sat_basic_dataset values (1, 1, 126, 1024, 555.666, 777.888, 999.000, true, 'q', 'some text', '2008-12-31', 'Jun 1,2008  09:59:59', 'Jun 1,2008 09:59:59 EST', '04:05:06', '04:05:06 EST', 'xxx'::varbyte);
+insert into sat_test_dataset.sat_basic_dataset values (2, -5, -126, -1024, -555.666, -777.888, -999.000, false, 'g', 'new text', '1987-10-10', 'Jun 21,2005  12:00:59', 'Oct 15,2003 09:59:59 EST', '04:05:00', '04:05:00 EST', 'yyy'::varbyte);
diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog.json
@@ -0,0 +1,19 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "sat_basic_dataset",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_cursor": null,
+        "default_cursor_field": null,
+        "source_defined_primary_key": [["id"]],
+        "namespace": null
+      },
+      "sync_mode": "full_refresh",
+      "cursor_field": null,
+      "destination_sync_mode": "append",
+      "primary_key": null
+    }
+  ]
+}
diff --git a/...yte-integrations/connectors/source-redshift/integration_tests/configured_catalog_inc.json b/...yte-integrations/connectors/source-redshift/integration_tests/configured_catalog_inc.json
@@ -0,0 +1,16 @@
+{
+  "streams": [
+    {
+      "stream": {
+        "name": "sat_basic_dataset",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["id"]
+      },
+      "sync_mode": "incremental",
+      "cursor_field": ["id"],
+      "destination_sync_mode": "append"
+    }
+  ]
+}
diff --git a/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json b/airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json
@@ -0,0 +1,2 @@
+{"stream": "sat_basic_dataset", "data": {"id":1,"test_column_1":1,"test_column_2":126,"test_column_3":1024,"test_column_4":556,"test_column_5":777.888,"test_column_6":999.0,"test_column_7":true,"test_column_8":"q","test_column_9":"some text","test_column_10":"2008-12-31T00:00:00Z","test_column_11":"2008-06-01T09:59:59.000000Z","test_column_12":"2008-06-01T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:06Z","test_column_14":"09:05:06+00","test_column_15":"787878"}, "emitted_at": 1669734903259 }
+{"stream": "sat_basic_dataset", "data": {"id":2,"test_column_1":-5,"test_column_2":-126,"test_column_3":-1024,"test_column_4":-556,"test_column_5":-777.888,"test_column_6":-999.0,"test_column_7":false,"test_column_8":"g","test_column_9":"new text","test_column_10":"1987-10-10T00:00:00Z","test_column_11":"2005-06-21T12:00:59.000000Z","test_column_12":"2003-10-15T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:00Z","test_column_14":"09:05:00+00","test_column_15":"797979"}, "emitted_at": 1669734903259 }