-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enable full SAT for the Redshift source (#19915)
* enable SAT for redshift * add bigger timeouts + specify catalogs (default works wrong) * new line * readme * new format Co-authored-by: Augustin <augustin@airbyte.io>
- Loading branch information
1 parent
dc71194
commit c2e4c3e
Showing
6 changed files
with
91 additions
and
3 deletions.
There are no files selected for viewing
34 changes: 31 additions & 3 deletions
34
airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,35 @@ | ||
# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) | ||
# for more information about how to configure these tests | ||
connector_image: airbyte/source-redshift:dev | ||
tests: | ||
acceptance_tests: | ||
spec: | ||
- spec_path: "src/test-integration/resources/expected_spec.json" | ||
config_path: "secrets/config.json" | ||
tests: | ||
- spec_path: "src/test-integration/resources/expected_spec.json" | ||
timeout_seconds: "1200" | ||
config_path: "secrets/config.json" | ||
connection: | ||
tests: | ||
- config_path: "secrets/config.json" | ||
timeout_seconds: "1200" | ||
status: "succeed" | ||
discovery: | ||
tests: | ||
- config_path: "secrets/config.json" | ||
timeout_seconds: "1200" | ||
basic_read: | ||
tests: | ||
- config_path: "secrets/config.json" | ||
timeout_seconds: "1200" | ||
configured_catalog_path: "integration_tests/configured_catalog.json" | ||
expect_records: | ||
path: "integration_tests/expected_records.json" | ||
full_refresh: | ||
tests: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "integration_tests/configured_catalog.json" | ||
timeout_seconds: "1200" | ||
incremental: | ||
tests: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "integration_tests/configured_catalog_inc.json" | ||
timeout_seconds: "1200" |
3 changes: 3 additions & 0 deletions
3
airbyte-integrations/connectors/source-redshift/integration_tests/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# Seeding the dataset | ||
You can find the SQL scripts in this folder if you need to create or fix the SAT dataset. | ||
For more instructions and information about valid scripts, please check this [doc](https://docs.google.com/document/d/1k5TvxaNhKdr44aJIHWWtLk14Tzd2gbNX-J8YNoTj8u0/edit#heading=h.ls9oiedt9wyy). |
20 changes: 20 additions & 0 deletions
20
airbyte-integrations/connectors/source-redshift/integration_tests/basic_dataset.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
CREATE TABLE IF NOT EXISTS sat_test_dataset.sat_basic_dataset ( | ||
id INTEGER, | ||
test_column_1 SMALLINT, | ||
test_column_2 INTEGER, | ||
test_column_3 BIGINT, | ||
test_column_4 DECIMAL, | ||
test_column_5 REAL, | ||
test_column_6 DOUBLE PRECISION, | ||
test_column_7 BOOLEAN, | ||
test_column_8 CHAR, | ||
test_column_9 VARCHAR, | ||
test_column_10 DATE, | ||
test_column_11 TIMESTAMP, | ||
test_column_12 TIMESTAMPTZ, | ||
test_column_13 TIME, | ||
test_column_14 TIMETZ, | ||
test_column_15 VARBYTE); | ||
|
||
insert into sat_test_dataset.sat_basic_dataset values (1, 1, 126, 1024, 555.666, 777.888, 999.000, true, 'q', 'some text', '2008-12-31', 'Jun 1,2008 09:59:59', 'Jun 1,2008 09:59:59 EST', '04:05:06', '04:05:06 EST', 'xxx'::varbyte); | ||
insert into sat_test_dataset.sat_basic_dataset values (2, -5, -126, -1024, -555.666, -777.888, -999.000, false, 'g', 'new text', '1987-10-10', 'Jun 21,2005 12:00:59', 'Oct 15,2003 09:59:59 EST', '04:05:00', '04:05:00 EST', 'yyy'::varbyte); |
19 changes: 19 additions & 0 deletions
19
airbyte-integrations/connectors/source-redshift/integration_tests/configured_catalog.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
{ | ||
"streams": [ | ||
{ | ||
"stream": { | ||
"name": "sat_basic_dataset", | ||
"json_schema": {}, | ||
"supported_sync_modes": ["full_refresh"], | ||
"source_defined_cursor": null, | ||
"default_cursor_field": null, | ||
"source_defined_primary_key": [["id"]], | ||
"namespace": null | ||
}, | ||
"sync_mode": "full_refresh", | ||
"cursor_field": null, | ||
"destination_sync_mode": "append", | ||
"primary_key": null | ||
} | ||
] | ||
} |
16 changes: 16 additions & 0 deletions
16
...yte-integrations/connectors/source-redshift/integration_tests/configured_catalog_inc.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
{ | ||
"streams": [ | ||
{ | ||
"stream": { | ||
"name": "sat_basic_dataset", | ||
"json_schema": {}, | ||
"supported_sync_modes": ["incremental"], | ||
"source_defined_cursor": true, | ||
"default_cursor_field": ["id"] | ||
}, | ||
"sync_mode": "incremental", | ||
"cursor_field": ["id"], | ||
"destination_sync_mode": "append" | ||
} | ||
] | ||
} |
2 changes: 2 additions & 0 deletions
2
airbyte-integrations/connectors/source-redshift/integration_tests/expected_records.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
{"stream": "sat_basic_dataset", "data": {"id":1,"test_column_1":1,"test_column_2":126,"test_column_3":1024,"test_column_4":556,"test_column_5":777.888,"test_column_6":999.0,"test_column_7":true,"test_column_8":"q","test_column_9":"some text","test_column_10":"2008-12-31T00:00:00Z","test_column_11":"2008-06-01T09:59:59.000000Z","test_column_12":"2008-06-01T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:06Z","test_column_14":"09:05:06+00","test_column_15":"787878"}, "emitted_at": 1669734903259 } | ||
{"stream": "sat_basic_dataset", "data": {"id":2,"test_column_1":-5,"test_column_2":-126,"test_column_3":-1024,"test_column_4":-556,"test_column_5":-777.888,"test_column_6":-999.0,"test_column_7":false,"test_column_8":"g","test_column_9":"new text","test_column_10":"1987-10-10T00:00:00Z","test_column_11":"2005-06-21T12:00:59.000000Z","test_column_12":"2003-10-15T14:59:59.000000Z","test_column_13":"1970-01-01T04:05:00Z","test_column_14":"09:05:00+00","test_column_15":"797979"}, "emitted_at": 1669734903259 } |