From aa872d001655eb35a973943a9241c8fea6f17904 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Wed, 20 Apr 2022 22:59:05 -0400 Subject: [PATCH] update various connectors to use YAML specs (#12230) * salesforce uses spec.yaml * stripe uses spec.yaml * revert unintended changes * bump stripe version * add yaml files to package_data * hubspot uses a spec.yaml * use double quotes in yaml files * google sheets uses spec.yaml * exhcange-rates uses spec.yaml * remove usage of default and enum in spec * bump hubspot cdk requirement * auto-bump connector version * auto-bump connector version * manually bump versions Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 10 +- .../src/main/resources/seed/source_specs.yaml | 16 +- .../source-exchange-rates/Dockerfile | 2 +- .../acceptance-test-config.yml | 2 +- .../connectors/source-exchange-rates/setup.py | 2 +- .../source_exchange_rates/spec.json | 33 ----- .../source_exchange_rates/spec.yaml | 37 +++++ .../source-google-sheets/Dockerfile | 2 +- .../connectors/source-google-sheets/README.md | 2 +- .../acceptance-test-config.yml | 2 +- .../google_sheets_source/spec.json | 89 ------------ .../google_sheets_source/spec.yaml | 83 +++++++++++ .../connectors/source-google-sheets/setup.py | 2 +- .../connectors/source-hubspot/Dockerfile | 2 +- .../connectors/source-hubspot/README.md | 2 +- .../source-hubspot/acceptance-test-config.yml | 2 +- .../connectors/source-hubspot/setup.py | 4 +- .../source-hubspot/source_hubspot/spec.json | 97 ------------- .../source-hubspot/source_hubspot/spec.yaml | 99 +++++++++++++ .../source-hubspot/source_hubspot/streams.py | 2 +- .../connectors/source-salesforce/Dockerfile | 2 +- .../connectors/source-salesforce/README.md | 2 +- .../acceptance-test-config.yml | 2 +- .../connectors/source-salesforce/setup.py | 2 +- .../source_salesforce/spec.json | 137 ------------------ .../source_salesforce/spec.yaml | 132 +++++++++++++++++ .../connectors/source-stripe/Dockerfile | 2 +- .../connectors/source-stripe/README.md | 2 +- .../source-stripe/acceptance-test-config.yml | 2 +- .../connectors/source-stripe/setup.py | 2 +- .../source-stripe/source_stripe/spec.json | 41 ------ .../source-stripe/source_stripe/spec.yaml | 47 ++++++ 32 files changed, 429 insertions(+), 434 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json create mode 100644 airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.yaml delete mode 100644 airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.json create mode 100644 airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml delete mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json create mode 100644 airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml delete mode 100644 airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.json create mode 100644 airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml delete mode 100644 airbyte-integrations/connectors/source-stripe/source_stripe/spec.json create mode 100644 airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 050406d2a99a..174ab8050a47 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -203,7 +203,7 @@ - name: Exchange Rates Api sourceDefinitionId: e2b40e36-aa0e-4bed-b41b-bcea6fa348b1 dockerRepository: airbyte/source-exchange-rates - dockerImageTag: 0.2.5 + dockerImageTag: 0.2.6 documentationUrl: https://docs.airbyte.io/integrations/sources/exchangeratesapi icon: exchangeratesapi.svg sourceType: api @@ -300,7 +300,7 @@ - name: Google Sheets sourceDefinitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 dockerRepository: airbyte/source-google-sheets - dockerImageTag: 0.2.11 + dockerImageTag: 0.2.12 documentationUrl: https://docs.airbyte.io/integrations/sources/google-sheets icon: google-sheets.svg sourceType: file @@ -341,7 +341,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.52 + dockerImageTag: 0.1.53 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api @@ -695,7 +695,7 @@ - name: Salesforce sourceDefinitionId: b117307c-14b6-41aa-9422-947e34922962 dockerRepository: airbyte/source-salesforce - dockerImageTag: 1.0.3 + dockerImageTag: 1.0.4 documentationUrl: https://docs.airbyte.io/integrations/sources/salesforce icon: salesforce.svg sourceType: api @@ -771,7 +771,7 @@ - name: Stripe sourceDefinitionId: e094cb9a-26de-4645-8761-65c0c425d1de dockerRepository: airbyte/source-stripe - dockerImageTag: 0.1.30 + dockerImageTag: 0.1.31 documentationUrl: https://docs.airbyte.io/integrations/sources/stripe icon: stripe.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 92932c037d3e..4a66caad8f49 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1709,7 +1709,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-exchange-rates:0.2.5" +- dockerImage: "airbyte/source-exchange-rates:0.2.6" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/exchangeratesapi" connectionSpecification: @@ -3142,7 +3142,7 @@ oauthFlowOutputParameters: - - "access_token" - - "refresh_token" -- dockerImage: "airbyte/source-google-sheets:0.2.11" +- dockerImage: "airbyte/source-google-sheets:0.2.12" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/google-sheets" connectionSpecification: @@ -3485,7 +3485,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-hubspot:0.1.52" +- dockerImage: "airbyte/source-hubspot:0.1.53" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" connectionSpecification: @@ -3523,9 +3523,6 @@ title: "Credentials Title" description: "Name of the credentials set" const: "OAuth Credentials" - enum: - - "OAuth Credentials" - default: "OAuth Credentials" order: 0 client_id: title: "Client ID" @@ -3564,9 +3561,6 @@ title: "Credentials title" description: "Name of the credentials set" const: "API Key Credentials" - enum: - - "API Key Credentials" - default: "API Key Credentials" order: 0 api_key: title: "API key" @@ -7357,7 +7351,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-salesforce:1.0.3" +- dockerImage: "airbyte/source-salesforce:1.0.4" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/salesforce" connectionSpecification: @@ -8279,7 +8273,7 @@ type: "string" path_in_connector_config: - "client_secret" -- dockerImage: "airbyte/source-stripe:0.1.30" +- dockerImage: "airbyte/source-stripe:0.1.31" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/stripe" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-exchange-rates/Dockerfile b/airbyte-integrations/connectors/source-exchange-rates/Dockerfile index 30094b221d43..656dd2f828d7 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/Dockerfile +++ b/airbyte-integrations/connectors/source-exchange-rates/Dockerfile @@ -16,5 +16,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=0.2.6 LABEL io.airbyte.name=airbyte/source-exchange-rates diff --git a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml index 1d43de5bb31d..457a11cf3d24 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml @@ -3,7 +3,7 @@ connector_image: airbyte/source-exchange-rates:dev tests: spec: - - spec_path: "source_exchange_rates/spec.json" + - spec_path: "source_exchange_rates/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-exchange-rates/setup.py b/airbyte-integrations/connectors/source-exchange-rates/setup.py index 7574bf07eb44..84ea1e232996 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/setup.py +++ b/airbyte-integrations/connectors/source-exchange-rates/setup.py @@ -11,6 +11,6 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, install_requires=["airbyte-cdk~=0.1", "pendulum>=2,<3"], ) diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json deleted file mode 100644 index 328d281ce8b5..000000000000 --- a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/exchangeratesapi", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "exchangeratesapi.io Source Spec", - "type": "object", - "required": ["start_date", "access_key"], - "additionalProperties": false, - "properties": { - "start_date": { - "type": "string", - "description": "Start getting data from that date.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["YYYY-MM-DD"] - }, - "access_key": { - "type": "string", - "description": "Your API Access Key. See here. The key is case sensitive.", - "airbyte_secret": true - }, - "base": { - "type": "string", - "description": "ISO reference currency. See here. Free plan doesn't support Source Currency Switching, default base currency is EUR", - "examples": ["EUR", "USD"] - }, - "ignore_weekends": { - "type": "boolean", - "description": "Ignore weekends? (Exchanges don't run on weekends)", - "default": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.yaml b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.yaml new file mode 100644 index 000000000000..adc25f06234e --- /dev/null +++ b/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/spec.yaml @@ -0,0 +1,37 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/exchangeratesapi +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: exchangeratesapi.io Source Spec + type: object + required: + - start_date + - access_key + additionalProperties: false + properties: + start_date: + type: string + description: Start getting data from that date. + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - YYYY-MM-DD + access_key: + type: string + description: >- + Your API Access Key. See here. The key is + case sensitive. + airbyte_secret: true + base: + type: string + description: >- + ISO reference currency. See here. + Free plan doesn't support Source Currency Switching, default base + currency is EUR + examples: + - EUR + - USD + ignore_weekends: + type: boolean + description: Ignore weekends? (Exchanges don't run on weekends) + default: true diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile index 1d97accfa942..343abcf8cab1 100644 --- a/airbyte-integrations/connectors/source-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/source-google-sheets/Dockerfile @@ -34,5 +34,5 @@ COPY google_sheets_source ./google_sheets_source ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.11 +LABEL io.airbyte.version=0.2.12 LABEL io.airbyte.name=airbyte/source-google-sheets diff --git a/airbyte-integrations/connectors/source-google-sheets/README.md b/airbyte-integrations/connectors/source-google-sheets/README.md index f6bc879a3909..504baef621a3 100644 --- a/airbyte-integrations/connectors/source-google-sheets/README.md +++ b/airbyte-integrations/connectors/source-google-sheets/README.md @@ -37,7 +37,7 @@ From the Airbyte repository root, run: #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-sheets) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_sheets/spec.json` file. +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_sheets/spec.yaml` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. diff --git a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml index aace1432f990..bb98e38ca722 100644 --- a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml @@ -3,7 +3,7 @@ connector_image: airbyte/source-google-sheets:dev tests: spec: - - spec_path: "google_sheets_source/spec.json" + - spec_path: "google_sheets_source/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.json b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.json deleted file mode 100644 index f89992b4143e..000000000000 --- a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-sheets", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Stripe Source Spec", - "type": "object", - "required": ["spreadsheet_id", "credentials"], - "additionalProperties": true, - "properties": { - "spreadsheet_id": { - "type": "string", - "title": "Spreadsheet Link", - "description": "The link to your spreadsheet. See this guide for more details.", - "examples": [ - "https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG-arw2xy4HR3D-dwUb/edit" - ] - }, - "credentials": { - "type": "object", - "title": "Authentication", - "description": "Google API Credentials for connecting to Google Sheets and Google Drive APIs", - "oneOf": [ - { - "title": "Authenticate via Google (OAuth)", - "type": "object", - "required": [ - "auth_type", - "client_id", - "client_secret", - "refresh_token" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "Client" - }, - "client_id": { - "title": "Client ID", - "type": "string", - "description": "The Client ID of your Google Sheets developer application.", - "airbyte_secret": true - }, - "client_secret": { - "title": "Client Secret", - "type": "string", - "description": "The Client Secret of your Google Sheets developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "type": "string", - "description": "The token for obtaining new access token.", - "airbyte_secret": true - } - } - }, - { - "title": "Service Account Key Authentication", - "type": "object", - "required": ["auth_type", "service_account_info"], - "properties": { - "auth_type": { - "type": "string", - "const": "Service" - }, - "service_account_info": { - "type": "string", - "title": "Service Account Information.", - "description": "The JSON key of the service account to use for authorization. See Setup Guide for more details", - "airbyte_secret": true, - "examples": [ - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" - ] - } - } - } - ] - } - } - }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", 0], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml new file mode 100644 index 000000000000..59c3ea2e094e --- /dev/null +++ b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml @@ -0,0 +1,83 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/google-sheets +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Stripe Source Spec + type: object + required: + - spreadsheet_id + - credentials + additionalProperties: true + properties: + spreadsheet_id: + type: string + title: Spreadsheet Link + description: >- + The link to your spreadsheet. See this + guide for more details. + examples: + - https://docs.google.com/spreadsheets/d/1hLd9Qqti3UyLXZB2aFfUWDT7BG-arw2xy4HR3D-dwUb/edit + credentials: + type: object + title: Authentication + description: >- + Google API Credentials for connecting to Google Sheets and Google Drive + APIs + oneOf: + - title: Authenticate via Google (OAuth) + type: object + required: + - auth_type + - client_id + - client_secret + - refresh_token + properties: + auth_type: + type: string + const: Client + client_id: + title: Client ID + type: string + description: The Client ID of your Google Sheets developer application. + airbyte_secret: true + client_secret: + title: Client Secret + type: string + description: The Client Secret of your Google Sheets developer application. + airbyte_secret: true + refresh_token: + title: Refresh Token + type: string + description: The token for obtaining new access token. + airbyte_secret: true + - title: Service Account Key Authentication + type: object + required: + - auth_type + - service_account_info + properties: + auth_type: + type: string + const: Service + service_account_info: + type: string + title: Service Account Information. + description: >- + The JSON key of the service account to use for authorization. + See Setup + Guide for more details + airbyte_secret: true + examples: + - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID, \"private_key_id\": YOUR_PRIVATE_KEY, ... }" +authSpecification: + auth_type: oauth2.0 + oauth2Specification: + rootObject: + - credentials + - 0 + oauthFlowInitParameters: + - ["client_id"] + - ["client_secret"] + oauthFlowOutputParameters: + - ["refresh_token"] diff --git a/airbyte-integrations/connectors/source-google-sheets/setup.py b/airbyte-integrations/connectors/source-google-sheets/setup.py index 0c65469b1d04..1ebeac130642 100644 --- a/airbyte-integrations/connectors/source-google-sheets/setup.py +++ b/airbyte-integrations/connectors/source-google-sheets/setup.py @@ -27,7 +27,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index e82ffce08597..d7b8bd20b572 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.52 +LABEL io.airbyte.version=0.1.53 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/README.md b/airbyte-integrations/connectors/source-hubspot/README.md index 6d914c99171a..ef1929193d0b 100644 --- a/airbyte-integrations/connectors/source-hubspot/README.md +++ b/airbyte-integrations/connectors/source-hubspot/README.md @@ -81,7 +81,7 @@ From the Airbyte repository root, run: #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hubspot) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubspot/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubspot/spec.yaml` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source hubspot test creds` and place them into `secrets/config.json`. diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index 47af46574217..6fe051146449 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -1,7 +1,7 @@ connector_image: airbyte/source-hubspot:dev tests: spec: - - spec_path: "source_hubspot/spec.json" + - spec_path: "source_hubspot/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index 39b5db3cb4ac..cf39bd556350 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1.49", + "airbyte-cdk~=0.1.55", "backoff==1.11.1", "pendulum==2.1.2", "requests==2.26.0", @@ -25,7 +25,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={"": ["*.json", "*,yaml", "schemas/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json deleted file mode 100644 index a6613d68e072..000000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "HubSpot Source Spec", - "type": "object", - "required": ["start_date", "credentials"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2017-01-25T00:00:00Z"] - }, - "credentials": { - "title": "Authentication mechanism", - "description": "Choose how to authenticate to HubSpot.", - "type": "object", - "oneOf": [ - { - "type": "object", - "title": "Authenticate via HubSpot (OAuth)", - "required": [ - "client_id", - "client_secret", - "refresh_token", - "credentials_title" - ], - "properties": { - "credentials_title": { - "type": "string", - "title": "Credentials Title", - "description": "Name of the credentials set", - "const": "OAuth Credentials", - "enum": ["OAuth Credentials"], - "default": "OAuth Credentials", - "order": 0 - }, - "client_id": { - "title": "Client ID", - "description": "The Client ID of your HubSpot developer application. See our docs if you need help finding this id.", - "type": "string", - "examples": ["123456789000"] - }, - "client_secret": { - "title": "Client Secret", - "description": "The Client Secret of your HubSpot developer application. See our docs if you need help finding this secret.", - "type": "string", - "examples": ["secret"], - "airbyte_secret": true - }, - "refresh_token": { - "title": "Refresh Token", - "description": "Refresh Token to renew the expired Access Token. See our docs if you need help generating the token.", - "type": "string", - "examples": ["refresh_token"], - "airbyte_secret": true - } - } - }, - { - "type": "object", - "title": "API key", - "required": ["api_key", "credentials_title"], - "properties": { - "credentials_title": { - "type": "string", - "title": "Credentials title", - "description": "Name of the credentials set", - "const": "API Key Credentials", - "enum": ["API Key Credentials"], - "default": "API Key Credentials", - "order": 0 - }, - "api_key": { - "title": "API key", - "description": "HubSpot API Key. See our docs if you need help finding this key.", - "type": "string", - "airbyte_secret": true - } - } - } - ] - } - } - }, - "authSpecification": { - "auth_type": "oauth2.0", - "oauth2Specification": { - "rootObject": ["credentials", "0"], - "oauthFlowInitParameters": [["client_id"], ["client_secret"]], - "oauthFlowOutputParameters": [["refresh_token"]] - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml new file mode 100644 index 000000000000..8ba82732e016 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml @@ -0,0 +1,99 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: HubSpot Source Spec + type: object + required: + - start_date + - credentials + additionalProperties: true + properties: + start_date: + type: string + title: Start Date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + description: >- + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before + this date will not be replicated. + examples: + - "2017-01-25T00:00:00Z" + credentials: + title: Authentication mechanism + description: Choose how to authenticate to HubSpot. + type: object + oneOf: + - type: object + title: Authenticate via HubSpot (OAuth) + required: + - client_id + - client_secret + - refresh_token + - credentials_title + properties: + credentials_title: + type: string + title: Credentials Title + description: Name of the credentials set + const: OAuth Credentials + order: 0 + client_id: + title: Client ID + description: >- + The Client ID of your HubSpot developer application. See our docs + if you need help finding this id. + type: string + examples: + - "123456789000" + client_secret: + title: Client Secret + description: >- + The Client Secret of your HubSpot developer application. See our + docs + if you need help finding this secret. + type: string + examples: + - secret + airbyte_secret: true + refresh_token: + title: Refresh Token + description: >- + Refresh Token to renew the expired Access Token. See our docs + if you need help generating the token. + type: string + examples: + - refresh_token + airbyte_secret: true + - type: object + title: API key + required: + - api_key + - credentials_title + properties: + credentials_title: + type: string + title: Credentials title + description: Name of the credentials set + const: API Key Credentials + order: 0 + api_key: + title: API key + description: >- + HubSpot API Key. See our docs + if you need help finding this key. + type: string + airbyte_secret: true +authSpecification: + auth_type: oauth2.0 + oauth2Specification: + rootObject: + - credentials + - "0" + oauthFlowInitParameters: + - ["client_id"] + - ["client_secret"] + oauthFlowOutputParameters: + - ["refresh_token"] diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index f0296c0cd21b..57221ba6d0df 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -145,7 +145,7 @@ def __init__(self, credentials: Mapping[str, Any]): elif credentials_title == "API Key Credentials": self._session.params["hapikey"] = credentials.get("api_key") else: - raise Exception("No supported `credentials_title` specified. See spec.json for references") + raise Exception("No supported `credentials_title` specified. See spec.yaml for references") self._session.headers = { "Content-Type": "application/json", diff --git a/airbyte-integrations/connectors/source-salesforce/Dockerfile b/airbyte-integrations/connectors/source-salesforce/Dockerfile index f45067228802..a87bfb6814a7 100644 --- a/airbyte-integrations/connectors/source-salesforce/Dockerfile +++ b/airbyte-integrations/connectors/source-salesforce/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.3 +LABEL io.airbyte.version=1.0.4 LABEL io.airbyte.name=airbyte/source-salesforce diff --git a/airbyte-integrations/connectors/source-salesforce/README.md b/airbyte-integrations/connectors/source-salesforce/README.md index d3df8e6f7a0b..bdeba0354ea7 100644 --- a/airbyte-integrations/connectors/source-salesforce/README.md +++ b/airbyte-integrations/connectors/source-salesforce/README.md @@ -39,7 +39,7 @@ To build using Gradle, from the Airbyte repository root, run: #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/salesforce) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_salesforce/spec.json` file. +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_salesforce/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. diff --git a/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml b/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml index 836c822330c1..95fde01f8999 100644 --- a/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml @@ -3,7 +3,7 @@ connector_image: airbyte/source-salesforce:dev tests: spec: - - spec_path: "source_salesforce/spec.json" + - spec_path: "source_salesforce/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-salesforce/setup.py b/airbyte-integrations/connectors/source-salesforce/setup.py index d86b8bd0f0eb..1d16e4376bba 100644 --- a/airbyte-integrations/connectors/source-salesforce/setup.py +++ b/airbyte-integrations/connectors/source-salesforce/setup.py @@ -16,7 +16,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.json b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.json deleted file mode 100644 index d7579bce5a1d..000000000000 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.json +++ /dev/null @@ -1,137 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/salesforce", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Salesforce Source Spec", - "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], - "additionalProperties": true, - "properties": { - "is_sandbox": { - "title": "Sandbox", - "description": "Whether the app is in a Salesforce sandbox or not. If you do not know what this is, assume it as false. We provide more info on this field in the docs.", - "type": "boolean", - "default": false, - "order": 1 - }, - "auth_type": { - "type": "string", - "const": "Client" - }, - "client_id": { - "title": "Client ID", - "description": "The Client ID of your Salesforce developer application.", - "type": "string", - "order": 2 - }, - "client_secret": { - "title": "Client Secret", - "description": "The Client Secret of your Salesforce developer application.", - "type": "string", - "airbyte_secret": true, - "order": 3 - }, - "refresh_token": { - "title": "Refresh Token", - "description": "Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow this guide to retrieve it.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "start_date": { - "title": "Start Date", - "description": "Date in the format 2017-01-25. Any data before this date will not be replicated. This field uses the \"updated\" field if available, otherwise the \"created\" fields if they are available for a stream. If not set, then by default all your data is replicated.", - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z|[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["2021-07-25", "2021-07-25T00:00:00Z"], - "order": 5 - }, - "streams_criteria": { - "type": "array", - "order": 6, - "items": { - "type": "object", - "required": ["criteria", "value"], - "properties": { - "criteria": { - "type": "string", - "title": "Search criteria", - "enum": [ - "starts with", - "ends with", - "contains", - "exacts", - "starts not with", - "ends not with", - "not contains", - "not exacts" - ], - "order": 1, - "default": "contains" - }, - "value": { - "type": "string", - "title": "Search value", - "order": 2 - } - } - }, - "title": "Salesforce Object filtering criteria (Optional)", - "description": "Add selection criteria for streams to read only the Salesforce objects which are relevant to you." - } - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "oauth_user_input_from_connector_config_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "is_sandbox": { - "type": "boolean", - "path_in_connector_config": ["is_sandbox"] - } - } - }, - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["client_secret"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml new file mode 100644 index 000000000000..afc3af769a7f --- /dev/null +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/spec.yaml @@ -0,0 +1,132 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/salesforce +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Salesforce Source Spec + type: object + required: + - client_id + - client_secret + - refresh_token + additionalProperties: true + properties: + is_sandbox: + title: Sandbox + description: >- + Whether the app is in a Salesforce sandbox or not. If you do not know + what this is, assume it as false. We provide more info on this field in + the docs. + type: boolean + default: false + order: 1 + auth_type: + type: string + const: Client + client_id: + title: Client ID + description: The Client ID of your Salesforce developer application. + type: string + order: 2 + client_secret: + title: Client Secret + description: The Client Secret of your Salesforce developer application. + type: string + airbyte_secret: true + order: 3 + refresh_token: + title: Refresh Token + description: >- + Salesforce Refresh Token used for Airbyte to access your Salesforce + account. If you don't know what this is, follow this guide + to retrieve it. + type: string + airbyte_secret: true + order: 4 + start_date: + title: Start Date + description: >- + Date in the format 2017-01-25. Any data before this date will not be + replicated. This field uses the "updated" field if available, otherwise + the "created" fields if they are available for a stream. If not set, + then by default all your data is replicated. + type: string + pattern: >- + ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z|[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - "2021-07-25" + - "2021-07-25T00:00:00Z" + order: 5 + streams_criteria: + type: array + order: 6 + items: + type: object + required: + - criteria + - value + properties: + criteria: + type: string + title: Search criteria + enum: + - starts with + - ends with + - contains + - exacts + - starts not with + - ends not with + - not contains + - not exacts + order: 1 + default: contains + value: + type: string + title: Search value + order: 2 + title: Salesforce Object filtering criteria (Optional) + description: >- + Add selection criteria for streams to read only the Salesforce objects + which are relevant to you. +advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - auth_type + predicate_value: Client + oauth_config_specification: + oauth_user_input_from_connector_config_specification: + type: object + additionalProperties: false + properties: + is_sandbox: + type: boolean + path_in_connector_config: + - is_sandbox + complete_oauth_output_specification: + type: object + additionalProperties: false + properties: + refresh_token: + type: string + path_in_connector_config: + - refresh_token + complete_oauth_server_input_specification: + type: object + additionalProperties: false + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + additionalProperties: false + properties: + client_id: + type: string + path_in_connector_config: + - client_id + client_secret: + type: string + path_in_connector_config: + - client_secret diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile index 9b6a706a22ec..01fa8f2f7b81 100644 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ b/airbyte-integrations/connectors/source-stripe/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.30 +LABEL io.airbyte.version=0.1.31 LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/README.md b/airbyte-integrations/connectors/source-stripe/README.md index 8a2e498164d2..8a8bc71daa84 100644 --- a/airbyte-integrations/connectors/source-stripe/README.md +++ b/airbyte-integrations/connectors/source-stripe/README.md @@ -39,7 +39,7 @@ To build using Gradle, from the Airbyte repository root, run: #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/stripe) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_stripe/spec.json` file. +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_stripe/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `sample_files/sample_config.json` for a sample config file. diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index b2e1beb29c36..b5d1cf6b9ec7 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -1,7 +1,7 @@ connector_image: airbyte/source-stripe:dev tests: spec: - - spec_path: "source_stripe/spec.json" + - spec_path: "source_stripe/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index e2a4aa01d4d6..1dbb2b968145 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -19,7 +19,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.json b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.json deleted file mode 100644 index 3696d24b1273..000000000000 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/stripe", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Stripe Source Spec", - "type": "object", - "required": ["client_secret", "account_id", "start_date"], - "additionalProperties": false, - "properties": { - "account_id": { - "type": "string", - "title": "Account ID", - "description": "Your Stripe account ID (starts with 'acct_', find yours here).", - "order": 0 - }, - "client_secret": { - "type": "string", - "title": "Secret Key", - "description": "Stripe API key (usually starts with 'sk_live_'; find yours here).", - "airbyte_secret": true, - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2017-01-25T00:00:00Z"], - "order": 2 - }, - "lookback_window_days": { - "type": "integer", - "title": "Lookback Window (in days)", - "default": 0, - "minimum": 0, - "description": "When set, the connector will always reload data from the past N days, where N is the value set here. This is useful if your data is updated after creation. More info here", - "order": 3 - } - } - } -} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml new file mode 100644 index 000000000000..67ec8e820b39 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml @@ -0,0 +1,47 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/stripe +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Stripe Source Spec + type: object + required: + - client_secret + - account_id + - start_date + additionalProperties: false + properties: + account_id: + type: string + title: Account ID + description: >- + Your Stripe account ID (starts with 'acct_', find yours here). + order: 0 + client_secret: + type: string + title: Secret Key + description: >- + Stripe API key (usually starts with 'sk_live_'; find yours here). + airbyte_secret: true + order: 1 + start_date: + type: string + title: Start Date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + description: >- + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before + this date will not be replicated. + examples: + - "2017-01-25T00:00:00Z" + order: 2 + lookback_window_days: + type: integer + title: Lookback Window (in days) + default: 0 + minimum: 0 + description: >- + When set, the connector will always reload data from the past N days, + where N is the value set here. This is useful if your data is updated + after creation. More info here + order: 3