From 1ea0369f5bcd63480a2fff42268bf59e506052dd Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 19:42:09 -0400 Subject: [PATCH 01/14] salesforce uses spec.yaml --- .../connectors/source-salesforce/Dockerfile | 2 +- .../connectors/source-salesforce/README.md | 2 +- .../acceptance-test-config.yml | 2 +- .../acceptance-test-docker.sh | 0 .../source_salesforce/spec.json | 137 ------------------ .../source_salesforce/spec.yaml | 132 +++++++++++++++++ 6 files changed, 135 insertions(+), 140 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-salesforce/acceptance-test-docker.sh delete mode 100644 airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.json create mode 100644 airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index f45067228802..a87bfb6814a7 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.3 +LABEL io.airbyte.version=1.0.4 LABEL io.airbyte.name=airbyte/source-salesforce diff --git a/airbyte-integrations/connectors/source-salesforce/README.md b/airbyte-integrations/connectors/source-salesforce/README.md index d3df8e6f7a0b..bdeba0354ea7 100644 --- a/airbyte-integrations/connectors/source-salesforce/README.md +++ b/airbyte-integrations/connectors/source-salesforce/README.md @@ -39,7 +39,7 @@ To build using Gradle, from the Airbyte repository root, run: #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/salesforce) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_salesforce/spec.json` file. +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_salesforce/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. diff --git a/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml b/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml index 836c822330c1..95fde01f8999 100644 --- a/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml @@ -3,7 +3,7 @@ connector_image: airbyte/source-salesforce:dev tests: spec: - - spec_path: "source_salesforce/spec.json" + - spec_path: "source_salesforce/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-salesforce/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-salesforce/acceptance-test-docker.sh old mode 100644 new mode 100755 diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.json b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.json deleted file mode 100644 index d7579bce5a1d..000000000000 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.json +++ /dev/null @@ -1,137 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/salesforce", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Salesforce Source Spec", - "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": true, - "properties": { - "is_sandbox": { - "title": "Sandbox", - "description": "Whether the app is in a Salesforce sandbox or not. If you do not know what this is, assume it as false. We provide more info on this field in the docs.", - "type": "boolean", - "default": false, - "order": 1 - }, - "auth_type": { - "type": "string", - "const": "Client" - }, - "client_id": { - "title": "Client ID", - "description": "The Client ID of your Salesforce developer application.", - "type": "string", - "order": 2 - }, - "client_secret": { - "title": "Client Secret", - "description": "The Client Secret of your Salesforce developer application.", - "type": "string", - "airbyte_secret": true, - "order": 3 - }, - "refresh_token": { - "title": "Refresh Token", - "description": "Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow this guide to retrieve it.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "start_date": { - "title": "Start Date", - "description": "Date in the format 2017-01-25. Any data before this date will not be replicated. This field uses the \"updated\" field if available, otherwise the \"created\" fields if they are available for a stream. If not set, then by default all your data is replicated.", - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z|[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["2021-07-25", "2021-07-25T00:00:00Z"], - "order": 5 - }, - "streams_criteria": { - "type": "array", - "order": 6, - "items": { - "type": "object", - "required": ["criteria", "value"], - "properties": { - "criteria": { - "type": "string", - "title": "Search criteria", - "enum": [ - "starts with", - "ends with", - "contains", - "exacts", - "starts not with", - "ends not with", - "not contains", - "not exacts" - ], - "order": 1, - "default": "contains" - }, - "value": { - "type": "string", - "title": "Search value", - "order": 2 - } - } - }, - "title": "Salesforce Object filtering criteria (Optional)", - "description": "Add selection criteria for streams to read only the Salesforce objects which are relevant to you." - } - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "is_sandbox": { - "type": "boolean", - "path_in_connector_config": ["is_sandbox"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["client_secret"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml new file mode 100644 index 000000000000..48003c944259 --- /dev/null +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml @@ -0,0 +1,132 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/salesforce +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Salesforce Source Spec + type: object + required: + - client_id + - client_secret + - refresh_token + additionalProperties: true + properties: + is_sandbox: + title: Sandbox + description: >- + Whether the app is in a Salesforce sandbox or not. If you do not know + what this is, assume it as false. We provide more info on this field in + the docs. + type: boolean + default: false + order: 1 + auth_type: + type: string + const: Client + client_id: + title: Client ID + description: The Client ID of your Salesforce developer application. + type: string + order: 2 + client_secret: + title: Client Secret + description: The Client Secret of your Salesforce developer application. + type: string + airbyte_secret: true + order: 3 + refresh_token: + title: Refresh Token + description: >- + Salesforce Refresh Token used for Airbyte to access your Salesforce + account. If you don't know what this is, follow this guide + to retrieve it. + type: string + airbyte_secret: true + order: 4 + start_date: + title: Start Date + description: >- + Date in the format 2017-01-25. Any data before this date will not be + replicated. This field uses the "updated" field if available, otherwise + the "created" fields if they are available for a stream. If not set, + then by default all your data is replicated. + type: string + pattern: >- + ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z|[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - '2021-07-25' + - '2021-07-25T00:00:00Z' + order: 5 + streams_criteria: + type: array + order: 6 + items: + type: object + required: + - criteria + - value + properties: + criteria: + type: string + title: Search criteria + enum: + - starts with + - ends with + - contains + - exacts + - starts not with + - ends not with + - not contains + - not exacts + order: 1 + default: contains + value: + type: string + title: Search value + order: 2 + title: Salesforce Object filtering criteria (Optional) + description: >- + Add selection criteria for streams to read only the Salesforce objects + which are relevant to you. +advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - auth_type + predicate_value: Client + oauth_config_specification: + oauth_user_input_from_connector_config_specification: + type: object + additionalProperties: false + properties: + is_sandbox: + type: boolean + path_in_connector_config: + - is_sandbox + complete_oauth_output_specification: + type: object + additionalProperties: false + properties: + refresh_token: + type: string + path_in_connector_config: + - refresh_token + complete_oauth_server_input_specification: + type: object + additionalProperties: false + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + additionalProperties: false + properties: + client_id: + type: string + path_in_connector_config: + - client_id + client_secret: + type: string + path_in_connector_config: + - client_secret From 73e8efb85533d294ed340cdc180b2f01a118179f Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 19:43:58 -0400 Subject: [PATCH 02/14] stripe uses spec.yaml --- .../connectors/source-stripe/README.md | 2 +- .../source-stripe/acceptance-test-config.yml | 68 +++++++++---------- .../source-stripe/acceptance-test-docker.sh | 4 +- .../source-stripe/source_stripe/spec.json | 41 ----------- .../source-stripe/source_stripe/spec.yaml | 47 +++++++++++++ 5 files changed, 84 insertions(+), 78 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh delete mode 100644 airbyte-integrations/connectors/source-stripe/source_stripe/spec.json create mode 100644 airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml diff --git a/airbyte-integrations/connectors/source-stripe/README.md b/airbyte-integrations/connectors/source-stripe/README.md index 8a2e498164d2..8a8bc71daa84 100644 --- a/airbyte-integrations/connectors/source-stripe/README.md +++ b/airbyte-integrations/connectors/source-stripe/README.md @@ -39,7 +39,7 @@ To build using Gradle, from the Airbyte repository root, run: #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/stripe) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_stripe/spec.json` file. +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_stripe/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index b2e1beb29c36..ae6650caf3fb 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -1,7 +1,7 @@ connector_image: airbyte/source-stripe:dev tests: spec: - - spec_path: "source_stripe/spec.json" + - spec_path: "source_stripe/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" @@ -9,36 +9,36 @@ tests: status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" - discovery: - - config_path: "secrets/config.json" - - config_path: "secrets/connected_account_config.json" - basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" - # TEST 1 - Reading catalog without invoice_line_items - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" - timeout_seconds: 3600 - # TEST 2 - Reading data from account that has no records for stream Disputes - - config_path: "secrets/connected_account_config.json" - configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" - timeout_seconds: 3600 - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - - config_path: "secrets/connected_account_config.json" - configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" - timeout_seconds: 3600 - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" - - config_path: "secrets/connected_account_config.json" - configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" - ignored_fields: - "invoices": - - invoice_pdf - - hosted_invoice_url +# discovery: +# - config_path: "secrets/config.json" +# - config_path: "secrets/connected_account_config.json" +# basic_read: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" +# # TEST 1 - Reading catalog without invoice_line_items +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" +# timeout_seconds: 3600 +# # TEST 2 - Reading data from account that has no records for stream Disputes +# - config_path: "secrets/connected_account_config.json" +# configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" +# timeout_seconds: 3600 +# incremental: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" +# - config_path: "secrets/connected_account_config.json" +# configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" +# full_refresh: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" +# timeout_seconds: 3600 +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" +# - config_path: "secrets/connected_account_config.json" +# configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" +# ignored_fields: +# "invoices": +# - invoice_pdf +# - hosted_invoice_url diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh old mode 100644 new mode 100755 index e4d8b1cef896..73c448859a3e --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh @@ -4,13 +4,13 @@ docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) # Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest +#docker pull airbyte/source-acceptance-test:latest # Run docker run --rm -it \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/tmp \ -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ + airbyte/source-acceptance-test:dev \ --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.json b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.json deleted file mode 100644 index 3696d24b1273..000000000000 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Stripe Source Spec", - "type": "object", - "required": ["client_secret", "account_id", "start_date"], - "additionalProperties": false, - "properties": { - "account_id": { - "type": "string", - "title": "Account ID", - "description": "Your Stripe account ID (starts with 'acct_', find yours here).", - "order": 0 - }, - "client_secret": { - "type": "string", - "title": "Secret Key", - "description": "Stripe API key (usually starts with 'sk_live_'; find yours here).", - "airbyte_secret": true, - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2017-01-25T00:00:00Z"], - "order": 2 - }, - "lookback_window_days": { - "type": "integer", - "title": "Lookback Window (in days)", - "default": 0, - "minimum": 0, - "description": "When set, the connector will always reload data from the past N days, where N is the value set here. This is useful if your data is updated after creation. More info here", - "order": 3 - } - } - } -} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml new file mode 100644 index 000000000000..2c2c0e05f30d --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml @@ -0,0 +1,47 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/stripe +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Stripe Source Spec + type: object + required: + - client_secret + - account_id + - start_date + additionalProperties: false + properties: + account_id: + type: string + title: Account ID + description: >- + Your Stripe account ID (starts with 'acct_', find yours here). + order: 0 + client_secret: + type: string + title: Secret Key + description: >- + Stripe API key (usually starts with 'sk_live_'; find yours here). + airbyte_secret: true + order: 1 + start_date: + type: string + title: Start Date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + description: >- + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before + this date will not be replicated. + examples: + - '2017-01-25T00:00:00Z' + order: 2 + lookback_window_days: + type: integer + title: Lookback Window (in days) + default: 0 + minimum: 0 + description: >- + When set, the connector will always reload data from the past N days, + where N is the value set here. This is useful if your data is updated + after creation. More info here + order: 3 From a350121da584c78a7021b05a9ee8d661d2d3a674 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 19:48:03 -0400 Subject: [PATCH 03/14] revert unintended changes --- .../acceptance-test-docker.sh | 0 .../source-stripe/acceptance-test-config.yml | 66 +++++++++---------- .../source-stripe/acceptance-test-docker.sh | 4 +- 3 files changed, 35 insertions(+), 35 deletions(-) mode change 100755 => 100644 airbyte-integrations/connectors/source-salesforce/acceptance-test-docker.sh mode change 100755 => 100644 airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh diff --git a/airbyte-integrations/connectors/source-salesforce/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-salesforce/acceptance-test-docker.sh old mode 100755 new mode 100644 diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index ae6650caf3fb..b5d1cf6b9ec7 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -9,36 +9,36 @@ tests: status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" -# discovery: -# - config_path: "secrets/config.json" -# - config_path: "secrets/connected_account_config.json" -# basic_read: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" -# # TEST 1 - Reading catalog without invoice_line_items -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" -# timeout_seconds: 3600 -# # TEST 2 - Reading data from account that has no records for stream Disputes -# - config_path: "secrets/connected_account_config.json" -# configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" -# timeout_seconds: 3600 -# incremental: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" -# - config_path: "secrets/connected_account_config.json" -# configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" -# full_refresh: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" -# timeout_seconds: 3600 -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" -# - config_path: "secrets/connected_account_config.json" -# configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" -# ignored_fields: -# "invoices": -# - invoice_pdf -# - hosted_invoice_url + discovery: + - config_path: "secrets/config.json" + - config_path: "secrets/connected_account_config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" + # TEST 1 - Reading catalog without invoice_line_items + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" + timeout_seconds: 3600 + # TEST 2 - Reading data from account that has no records for stream Disputes + - config_path: "secrets/connected_account_config.json" + configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" + timeout_seconds: 3600 + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/connected_account_config.json" + configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/non_invoice_line_items_catalog.json" + timeout_seconds: 3600 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/full_refresh_configured_catalog.json" + - config_path: "secrets/connected_account_config.json" + configured_catalog_path: "integration_tests/connected_account_configured_catalog.json" + ignored_fields: + "invoices": + - invoice_pdf + - hosted_invoice_url diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh old mode 100755 new mode 100644 index 73c448859a3e..e4d8b1cef896 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh @@ -4,13 +4,13 @@ docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) # Pull latest acctest image -#docker pull airbyte/source-acceptance-test:latest +docker pull airbyte/source-acceptance-test:latest # Run docker run --rm -it \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /tmp:/tmp \ -v $(pwd):/test_input \ - airbyte/source-acceptance-test:dev \ + airbyte/source-acceptance-test \ --acceptance-test-config /test_input From d7b3bd7c93047fd5cfe26eef347e0940ca4dcefd Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 19:49:15 -0400 Subject: [PATCH 04/14] bump stripe version --- airbyte-integrations/connectors/source-stripe/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index 9b6a706a22ec..01fa8f2f7b81 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.30 +LABEL io.airbyte.version=0.1.31 LABEL io.airbyte.name=airbyte/source-stripe From 87e86096e50bd24151a6f42fc59abfc7f54197f9 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 19:55:53 -0400 Subject: [PATCH 05/14] add yaml files to package_data --- airbyte-integrations/connectors/source-salesforce/setup.py | 2 +- airbyte-integrations/connectors/source-stripe/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-salesforce/setup.py b/airbyte-integrations/connectors/source-salesforce/setup.py index d86b8bd0f0eb..1d16e4376bba 100644 --- a/airbyte-integrations/connectors/source-salesforce/setup.py +++ b/airbyte-integrations/connectors/source-salesforce/setup.py @@ -16,7 +16,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index e2a4aa01d4d6..1dbb2b968145 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -19,7 +19,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, From f2cf0e61f3e64e6f2e6942492a18be5768f218dc Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 20:12:04 -0400 Subject: [PATCH 06/14] hubspot uses a spec.yaml --- .../connectors/source-hubspot/Dockerfile | 2 +- .../connectors/source-hubspot/README.md | 2 +- .../source-hubspot/acceptance-test-config.yml | 2 +- .../connectors/source-hubspot/setup.py | 2 +- .../source-hubspot/source_hubspot/spec.json | 97 ---------------- .../source-hubspot/source_hubspot/spec.yaml | 105 ++++++++++++++++++ .../source-hubspot/source_hubspot/streams.py | 2 +- 7 files changed, 110 insertions(+), 102 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json create mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index e82ffce08597..d7b8bd20b572 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.52 +LABEL io.airbyte.version=0.1.53 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/README.md b/airbyte-integrations/connectors/source-hubspot/README.md index 6d914c99171a..ef1929193d0b 100644 --- a/airbyte-integrations/connectors/source-hubspot/README.md +++ b/airbyte-integrations/connectors/source-hubspot/README.md @@ -81,7 +81,7 @@ From the Airbyte repository root, run: #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hubspot) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubspot/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubspot/spec.yaml` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source hubspot test creds` and place them into `secrets/config.json`. diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index 47af46574217..6fe051146449 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -1,7 +1,7 @@ connector_image: airbyte/source-hubspot:dev tests: spec: - - spec_path: "source_hubspot/spec.json" + - spec_path: "source_hubspot/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index 39b5db3cb4ac..d721a1d01071 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -25,7 +25,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={"": ["*.json", "*,yaml", "schemas/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json deleted file mode 100644 index a6613d68e072..000000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "HubSpot Source Spec", - "type": "object", - "required": ["start_date", "credentials"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2017-01-25T00:00:00Z"] - }, - "credentials": { - "title": "Authentication mechanism", - "description": "Choose how to authenticate to HubSpot.", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "Authenticate via HubSpot (OAuth)", - "required": [ - "client_id", - "client_secret", - "refresh_token", - "credentials_title" - ], - "properties": { - "credentials_title": { - "type": "string", - "title": "Credentials Title", - "description": "Name of the credentials set", - "const": "OAuth Credentials", - "enum": ["OAuth Credentials"], - "default": "OAuth Credentials", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "description": "The Client ID of your HubSpot developer application. See our docs if you need help finding this id.", - "type": "string", - "examples": ["123456789000"] - }, - "client_secret": { - "title": "Client Secret", - "description": "The Client Secret of your HubSpot developer application. See our docs if you need help finding this secret.", - "type": "string", - "examples": ["secret"], - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "description": "Refresh Token to renew the expired Access Token. See our docs if you need help generating the token.", - "type": "string", - "examples": ["refresh_token"], - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API key", - "required": ["api_key", "credentials_title"], - "properties": { - "credentials_title": { - "type": "string", - "title": "Credentials title", - "description": "Name of the credentials set", - "const": "API Key Credentials", - "enum": ["API Key Credentials"], - "default": "API Key Credentials", - "order": 0 - }, - "api_key": { - "title": "API key", - "description": "HubSpot API Key. See our docs if you need help finding this key.", - "type": "string", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml new file mode 100644 index 000000000000..3b8add68db1d --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml @@ -0,0 +1,105 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: HubSpot Source Spec + type: object + required: + - start_date + - credentials + additionalProperties: true + properties: + start_date: + type: string + title: Start Date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + description: >- + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before + this date will not be replicated. + examples: + - "2017-01-25T00:00:00Z" + credentials: + title: Authentication mechanism + description: Choose how to authenticate to HubSpot. + type: object + oneOf: + - type: object + title: Authenticate via HubSpot (OAuth) + required: + - client_id + - client_secret + - refresh_token + - credentials_title + properties: + credentials_title: + type: string + title: Credentials Title + description: Name of the credentials set + const: OAuth Credentials + enum: + - OAuth Credentials + default: OAuth Credentials + order: 0 + client_id: + title: Client ID + description: >- + The Client ID of your HubSpot developer application. See our docs + if you need help finding this id. + type: string + examples: + - "123456789000" + client_secret: + title: Client Secret + description: >- + The Client Secret of your HubSpot developer application. See our + docs + if you need help finding this secret. + type: string + examples: + - secret + airbyte_secret: true + refresh_token: + title: Refresh Token + description: >- + Refresh Token to renew the expired Access Token. See our docs + if you need help generating the token. + type: string + examples: + - refresh_token + airbyte_secret: true + - type: object + title: API key + required: + - api_key + - credentials_title + properties: + credentials_title: + type: string + title: Credentials title + description: Name of the credentials set + const: API Key Credentials + enum: + - API Key Credentials + default: API Key Credentials + order: 0 + api_key: + title: API key + description: >- + HubSpot API Key. See our docs + if you need help finding this key. + type: string + airbyte_secret: true +authSpecification: + auth_type: oauth2.0 + oauth2Specification: + rootObject: + - credentials + - "0" + oauthFlowInitParameters: + - ["client_id"] + - ["client_secret"] + oauthFlowOutputParameters: + - ["refresh_token"] diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index f0296c0cd21b..57221ba6d0df 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -145,7 +145,7 @@ def __init__(self, credentials: Mapping[str, Any]): elif credentials_title == "API Key Credentials": self._session.params["hapikey"] = credentials.get("api_key") else: - raise Exception("No supported `credentials_title` specified. See spec.json for references") + raise Exception("No supported `credentials_title` specified. See spec.yaml for references") self._session.headers = { "Content-Type": "application/json", From 8f30a4ef78a49d7785a9a0a8947e564dd69ce031 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 20:15:01 -0400 Subject: [PATCH 07/14] use double quotes in yaml files --- .../connectors/source-salesforce/source_salesforce/spec.yaml | 4 ++-- .../connectors/source-stripe/source_stripe/spec.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml index 48003c944259..afc3af769a7f 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml @@ -54,8 +54,8 @@ connectionSpecification: pattern: >- ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z|[0-9]{4}-[0-9]{2}-[0-9]{2}$ examples: - - '2021-07-25' - - '2021-07-25T00:00:00Z' + - "2021-07-25" + - "2021-07-25T00:00:00Z" order: 5 streams_criteria: type: array diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml index 2c2c0e05f30d..67ec8e820b39 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml @@ -32,7 +32,7 @@ connectionSpecification: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. examples: - - '2017-01-25T00:00:00Z' + - "2017-01-25T00:00:00Z" order: 2 lookback_window_days: type: integer From dd29c0995793374f00a8a9ef7e9de03d8180e9dd Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 20:21:10 -0400 Subject: [PATCH 08/14] google sheets uses spec.yaml --- .../source-google-sheets/Dockerfile | 2 +- .../connectors/source-google-sheets/README.md | 2 +- .../acceptance-test-config.yml | 2 +- .../google_sheets_source/spec.json | 89 ------------------- .../google_sheets_source/spec.yaml | 83 +++++++++++++++++ .../connectors/source-google-sheets/setup.py | 2 +- 6 files changed, 87 insertions(+), 93 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.json create mode 100644 airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile index 1d97accfa942..343abcf8cab1 100644 --- a/airbyte-integrations/connectors/source-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/source-google-sheets/Dockerfile @@ -34,5 +34,5 @@ COPY google_sheets_source ./google_sheets_source ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.11 +LABEL io.airbyte.version=0.2.12 LABEL io.airbyte.name=airbyte/source-google-sheets diff --git a/airbyte-integrations/connectors/source-google-sheets/README.md b/airbyte-integrations/connectors/source-google-sheets/README.md index f6bc879a3909..504baef621a3 100644 --- a/airbyte-integrations/connectors/source-google-sheets/README.md +++ b/airbyte-integrations/connectors/source-google-sheets/README.md @@ -37,7 +37,7 @@ From the Airbyte repository root, run: #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-sheets) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_sheets/spec.json` file. +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_sheets/spec.yaml` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. diff --git a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml index aace1432f990..bb98e38ca722 100644 --- a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml @@ -3,7 +3,7 @@ connector_image: airbyte/source-google-sheets:dev tests: spec: - - spec_path: "google_sheets_source/spec.json" + - spec_path: "google_sheets_source/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.json b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.json deleted file mode 100644 index f89992b4143e..000000000000 --- a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-sheets", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Stripe Source Spec", - "type": "object", - "required": ["spreadsheet_id", "credentials"], - "additionalProperties": true, - "properties": { - "spreadsheet_id": { - "type": "string", - "title": "Spreadsheet Link", - "description": "The link to your spreadsheet. See this guide for more details.", - "examples": [ - "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG-arw2xy4HR3D-dwUb/edit" - ] - }, - "credentials": { - "type": "object", - "title": "Authentication", - "description": "Google API Credentials for connecting to Google Sheets and Google Drive APIs", - "oneOf": [ - { - "title": "Authenticate via Google (OAuth)", - "type": "object", - "required": [ - "auth_type", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "Client" - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Google Sheets developer application.", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Google Sheets developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "The token for obtaining new access token.", - "airbyte_secret": true - } - } - }, - { - "title": "Service Account Key Authentication", - "type": "object", - "required": ["auth_type", "service_account_info"], - "properties": { - "auth_type": { - "type": "string", - "const": "Service" - }, - "service_account_info": { - "type": "string", - "title": "Service Account Information.", - "description": "The JSON key of the service account to use for authorization. See Setup Guide for more details", - "airbyte_secret": true, - "examples": [ - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - ] - } - } - } - ] - } - } - }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 0], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml new file mode 100644 index 000000000000..59c3ea2e094e --- /dev/null +++ b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml @@ -0,0 +1,83 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/google-sheets +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Stripe Source Spec + type: object + required: + - spreadsheet_id + - credentials + additionalProperties: true + properties: + spreadsheet_id: + type: string + title: Spreadsheet Link + description: >- + The link to your spreadsheet. See this + guide for more details. + examples: + - https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG-arw2xy4HR3D-dwUb/edit + credentials: + type: object + title: Authentication + description: >- + Google API Credentials for connecting to Google Sheets and Google Drive + APIs + oneOf: + - title: Authenticate via Google (OAuth) + type: object + required: + - auth_type + - client_id + - client_secret + - refresh_token + properties: + auth_type: + type: string + const: Client + client_id: + title: Client ID + type: string + description: The Client ID of your Google Sheets developer application. + airbyte_secret: true + client_secret: + title: Client Secret + type: string + description: The Client Secret of your Google Sheets developer application. + airbyte_secret: true + refresh_token: + title: Refresh Token + type: string + description: The token for obtaining new access token. + airbyte_secret: true + - title: Service Account Key Authentication + type: object + required: + - auth_type + - service_account_info + properties: + auth_type: + type: string + const: Service + service_account_info: + type: string + title: Service Account Information. + description: >- + The JSON key of the service account to use for authorization. + See Setup + Guide for more details + airbyte_secret: true + examples: + - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" +authSpecification: + auth_type: oauth2.0 + oauth2Specification: + rootObject: + - credentials + - 0 + oauthFlowInitParameters: + - ["client_id"] + - ["client_secret"] + oauthFlowOutputParameters: + - ["refresh_token"] diff --git a/airbyte-integrations/connectors/source-google-sheets/setup.py b/airbyte-integrations/connectors/source-google-sheets/setup.py index 0c65469b1d04..1ebeac130642 100644 --- a/airbyte-integrations/connectors/source-google-sheets/setup.py +++ b/airbyte-integrations/connectors/source-google-sheets/setup.py @@ -27,7 +27,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, From ff1d757b150c03dd46a099215de8ebdd11b8682f Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 20:41:39 -0400 Subject: [PATCH 09/14] exhcange-rates uses spec.yaml --- .../source-exchange-rates/Dockerfile | 2 +- .../acceptance-test-config.yml | 2 +- .../connectors/source-exchange-rates/setup.py | 2 +- .../source_exchange_rates/spec.json | 33 ----------------- .../source_exchange_rates/spec.yaml | 37 +++++++++++++++++++ 5 files changed, 40 insertions(+), 36 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json create mode 100644 airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.yaml diff --git a/airbyte-integrations/connectors/source-exchange-rates/Dockerfile b/airbyte-integrations/connectors/source-exchange-rates/Dockerfile index 30094b221d43..656dd2f828d7 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/Dockerfile +++ b/airbyte-integrations/connectors/source-exchange-rates/Dockerfile @@ -16,5 +16,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=0.2.6 LABEL io.airbyte.name=airbyte/source-exchange-rates diff --git a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml index 1d43de5bb31d..457a11cf3d24 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml @@ -3,7 +3,7 @@ connector_image: airbyte/source-exchange-rates:dev tests: spec: - - spec_path: "source_exchange_rates/spec.json" + - spec_path: "source_exchange_rates/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-exchange-rates/setup.py b/airbyte-integrations/connectors/source-exchange-rates/setup.py index 7574bf07eb44..84ea1e232996 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/setup.py +++ b/airbyte-integrations/connectors/source-exchange-rates/setup.py @@ -11,6 +11,6 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, install_requires=["airbyte-cdk~=0.1", "pendulum>=2,<3"], ) diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json deleted file mode 100644 index 328d281ce8b5..000000000000 --- a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/exchangeratesapi", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "exchangeratesapi.io Source Spec", - "type": "object", - "required": ["start_date", "access_key"], - "additionalProperties": false, - "properties": { - "start_date": { - "type": "string", - "description": "Start getting data from that date.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["YYYY-MM-DD"] - }, - "access_key": { - "type": "string", - "description": "Your API Access Key. See here. The key is case sensitive.", - "airbyte_secret": true - }, - "base": { - "type": "string", - "description": "ISO reference currency. See here. Free plan doesn't support Source Currency Switching, default base currency is EUR", - "examples": ["EUR", "USD"] - }, - "ignore_weekends": { - "type": "boolean", - "description": "Ignore weekends? (Exchanges don't run on weekends)", - "default": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.yaml b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.yaml new file mode 100644 index 000000000000..adc25f06234e --- /dev/null +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.yaml @@ -0,0 +1,37 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/exchangeratesapi +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: exchangeratesapi.io Source Spec + type: object + required: + - start_date + - access_key + additionalProperties: false + properties: + start_date: + type: string + description: Start getting data from that date. + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - YYYY-MM-DD + access_key: + type: string + description: >- + Your API Access Key. See here. The key is + case sensitive. + airbyte_secret: true + base: + type: string + description: >- + ISO reference currency. See here. + Free plan doesn't support Source Currency Switching, default base + currency is EUR + examples: + - EUR + - USD + ignore_weekends: + type: boolean + description: Ignore weekends? (Exchanges don't run on weekends) + default: true From 741234b3363c2987a0f5464d0135055c2273d8d9 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 20:47:39 -0400 Subject: [PATCH 10/14] remove usage of default and enum in spec --- .../connectors/source-hubspot/source_hubspot/spec.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml index 3b8add68db1d..8ba82732e016 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml @@ -35,9 +35,6 @@ connectionSpecification: title: Credentials Title description: Name of the credentials set const: OAuth Credentials - enum: - - OAuth Credentials - default: OAuth Credentials order: 0 client_id: title: Client ID @@ -80,9 +77,6 @@ connectionSpecification: title: Credentials title description: Name of the credentials set const: API Key Credentials - enum: - - API Key Credentials - default: API Key Credentials order: 0 api_key: title: API key From 110e704c932a566895a3bab69fe09bb463c8c065 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 21:03:30 -0400 Subject: [PATCH 11/14] bump hubspot cdk requirement --- airbyte-integrations/connectors/source-hubspot/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index d721a1d01071..cf39bd556350 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1.49", + "airbyte-cdk~=0.1.55", "backoff==1.11.1", "pendulum==2.1.2", "requests==2.26.0", From 9e8a94559f3ef5aaf1f08852ea645754e613e929 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 21 Apr 2022 02:03:27 +0000 Subject: [PATCH 12/14] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 050406d2a99a..5ec63dc281e2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -695,7 +695,7 @@ - name: Salesforce sourceDefinitionId: b117307c-14b6-41aa-9422-947e34922962 dockerRepository: airbyte/source-salesforce - dockerImageTag: 1.0.3 + dockerImageTag: 1.0.4 documentationUrl: https://docs.airbyte.io/integrations/sources/salesforce icon: salesforce.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 92932c037d3e..0def0fc4de61 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7357,7 +7357,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-salesforce:1.0.3" +- dockerImage: "airbyte/source-salesforce:1.0.4" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/salesforce" connectionSpecification: From e579adfef9a3ec500b03f214264d52239426f939 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Thu, 21 Apr 2022 02:26:08 +0000 Subject: [PATCH 13/14] auto-bump connector version --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5ec63dc281e2..aff1993aed31 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -341,7 +341,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.52 + dockerImageTag: 0.1.53 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 0def0fc4de61..8df5aebc560c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3485,7 +3485,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-hubspot:0.1.52" +- dockerImage: "airbyte/source-hubspot:0.1.53" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" connectionSpecification: @@ -3523,9 +3523,6 @@ title: "Credentials Title" description: "Name of the credentials set" const: "OAuth Credentials" - enum: - - "OAuth Credentials" - default: "OAuth Credentials" order: 0 client_id: title: "Client ID" @@ -3564,9 +3561,6 @@ title: "Credentials title" description: "Name of the credentials set" const: "API Key Credentials" - enum: - - "API Key Credentials" - default: "API Key Credentials" order: 0 api_key: title: "API key" From 16bc7991795c1f526cea5aa1d7c77646568d713e Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 22:35:23 -0400 Subject: [PATCH 14/14] manually bump versions --- .../init/src/main/resources/seed/source_definitions.yaml | 6 +++--- .../init/src/main/resources/seed/source_specs.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index aff1993aed31..174ab8050a47 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -203,7 +203,7 @@ - name: Exchange Rates Api sourceDefinitionId: e2b40e36-aa0e-4bed-b41b-bcea6fa348b1 dockerRepository: airbyte/source-exchange-rates - dockerImageTag: 0.2.5 + dockerImageTag: 0.2.6 documentationUrl: https://docs.airbyte.io/integrations/sources/exchangeratesapi icon: exchangeratesapi.svg sourceType: api @@ -300,7 +300,7 @@ - name: Google Sheets sourceDefinitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 dockerRepository: airbyte/source-google-sheets - dockerImageTag: 0.2.11 + dockerImageTag: 0.2.12 documentationUrl: https://docs.airbyte.io/integrations/sources/google-sheets icon: google-sheets.svg sourceType: file @@ -771,7 +771,7 @@ - name: Stripe sourceDefinitionId: e094cb9a-26de-4645-8761-65c0c425d1de dockerRepository: airbyte/source-stripe - dockerImageTag: 0.1.30 + dockerImageTag: 0.1.31 documentationUrl: https://docs.airbyte.io/integrations/sources/stripe icon: stripe.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 8df5aebc560c..4a66caad8f49 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1709,7 +1709,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-exchange-rates:0.2.5" +- dockerImage: "airbyte/source-exchange-rates:0.2.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/exchangeratesapi" connectionSpecification: @@ -3142,7 +3142,7 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" -- dockerImage: "airbyte/source-google-sheets:0.2.11" +- dockerImage: "airbyte/source-google-sheets:0.2.12" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/google-sheets" connectionSpecification: @@ -8273,7 +8273,7 @@ type: "string" path_in_connector_config: - "client_secret" -- dockerImage: "airbyte/source-stripe:0.1.30" +- dockerImage: "airbyte/source-stripe:0.1.31" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/stripe" connectionSpecification: