From 0043874c9a28a662ea04bcdc36ccf845ca5a3a19 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Wed, 26 May 2021 15:22:20 +0300 Subject: [PATCH 01/19] added filter of streams and deleted duplicated title in spec.json --- .../source_googleanalytics_singer/source.py | 13 +++++++++++-- .../source_googleanalytics_singer/spec.json | 1 - 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py index 0780c9d6a0e6..6de9bb29ae7f 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py @@ -29,6 +29,7 @@ from pathlib import Path from typing import List +from airbyte_protocol import ConfiguredAirbyteCatalog from base_singer import AirbyteLogger, BaseSingerSource from jsonschema.validators import Draft4Validator from tap_google_analytics import GAClient @@ -42,6 +43,7 @@ class GoogleAnalyticsSingerSource(BaseSingerSource): tap_cmd = "tap-google-analytics" tap_name = "Google Analytics API" api_error = Exception + reports_to_read = [] # can be overridden to change an input config def configure(self, raw_config: json, temp_dir: str) -> json: @@ -59,19 +61,26 @@ def _validate_custom_reports(self, custom_reports_data: List[dict]): error_messages.append(error.message) raise Exception("An error occurred during custom_reports data validation: " + "; ".join(error_messages)) + def read_catalog(self, catalog_path: str) -> ConfiguredAirbyteCatalog: + catalog = ConfiguredAirbyteCatalog.parse_obj(self.read_config(catalog_path)) + if not self.reports_to_read: + self.reports_to_read = [i.stream.name for i in catalog.streams] + return catalog_path + def _get_reports_file_path(self, temp_dir: str, custom_reports_data: List[dict]) -> str: report_definition = ( json.loads(pkgutil.get_data("tap_google_analytics", "defaults/default_report_definition.json")) + custom_reports_data ) + new_report_definition = [i for i in report_definition if i["name"] in self.reports_to_read] custom_reports = os.path.join(temp_dir, "custom_reports.json") with open(custom_reports, "w") as file: - file.write(json.dumps(report_definition)) + file.write(json.dumps(new_report_definition)) + return custom_reports def _check_custom_reports(self, config: dict = None, config_path: str = None): if config_path: config = self.read_config(config_path) - custom_reports = config.pop("custom_reports") if custom_reports.strip() and json.loads(custom_reports): custom_reports_data = json.loads(custom_reports) diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/spec.json b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/spec.json index 40c7467a1e06..164651b7645b 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/spec.json +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/spec.json @@ -27,7 +27,6 @@ "custom_reports": { "title": "Custom Reports", "type": "string", - "title": "Custom Reports", "description": "A JSON array describing the custom reports you want to sync from GA. Check out the docs to get more information about this field." } } From 21b52b01098ec3692f7375dfb05de9e4866d88e9 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Fri, 28 May 2021 13:48:07 +0300 Subject: [PATCH 02/19] fixed discover returns empty streams --- .../bases/base-python/base_python/__init__.py | 0 .../source_facebook_marketing/client/client.py | 0 .../connectors/source-googleanalytics-singer/main_dev.py | 0 .../source_googleanalytics_singer/source.py | 8 +++++--- .../source-smartsheets/source_smartsheets/source.py | 0 5 files changed, 5 insertions(+), 3 deletions(-) mode change 100644 => 100755 airbyte-integrations/bases/base-python/base_python/__init__.py mode change 100644 => 100755 airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/client.py mode change 100644 => 100755 airbyte-integrations/connectors/source-googleanalytics-singer/main_dev.py mode change 100644 => 100755 airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py mode change 100644 => 100755 airbyte-integrations/connectors/source-smartsheets/source_smartsheets/source.py diff --git a/airbyte-integrations/bases/base-python/base_python/__init__.py b/airbyte-integrations/bases/base-python/base_python/__init__.py old mode 100644 new mode 100755 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/client.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/client.py old mode 100644 new mode 100755 diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/main_dev.py b/airbyte-integrations/connectors/source-googleanalytics-singer/main_dev.py old mode 100644 new mode 100755 diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py old mode 100644 new mode 100755 index 6de9bb29ae7f..3c16e1b012e4 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py @@ -43,7 +43,7 @@ class GoogleAnalyticsSingerSource(BaseSingerSource): tap_cmd = "tap-google-analytics" tap_name = "Google Analytics API" api_error = Exception - reports_to_read = [] + reports_to_read = None # can be overridden to change an input config def configure(self, raw_config: json, temp_dir: str) -> json: @@ -71,10 +71,12 @@ def _get_reports_file_path(self, temp_dir: str, custom_reports_data: List[dict]) report_definition = ( json.loads(pkgutil.get_data("tap_google_analytics", "defaults/default_report_definition.json")) + custom_reports_data ) - new_report_definition = [i for i in report_definition if i["name"] in self.reports_to_read] + if self.reports_to_read is not None: + report_definition = [i for i in report_definition if i["name"] in self.reports_to_read] + custom_reports = os.path.join(temp_dir, "custom_reports.json") with open(custom_reports, "w") as file: - file.write(json.dumps(new_report_definition)) + file.write(json.dumps(report_definition)) return custom_reports diff --git a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/source.py b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/source.py old mode 100644 new mode 100755 From e81a4650f75a52f99de50546f387054fb4539c94 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Wed, 2 Jun 2021 14:59:17 +0300 Subject: [PATCH 03/19] pr changes --- .../source_googleanalytics_singer/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py index 3c16e1b012e4..771dceead871 100755 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py @@ -71,7 +71,7 @@ def _get_reports_file_path(self, temp_dir: str, custom_reports_data: List[dict]) report_definition = ( json.loads(pkgutil.get_data("tap_google_analytics", "defaults/default_report_definition.json")) + custom_reports_data ) - if self.reports_to_read is not None: + if self.reports_to_read: report_definition = [i for i in report_definition if i["name"] in self.reports_to_read] custom_reports = os.path.join(temp_dir, "custom_reports.json") From c5321ce6200fc14d2656322bc5b5d015c88687ab Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Thu, 3 Jun 2021 17:05:40 +0300 Subject: [PATCH 04/19] chmod on files changed --- airbyte-integrations/bases/base-python/base_python/__init__.py | 0 .../source_facebook_marketing/client/client.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 airbyte-integrations/bases/base-python/base_python/__init__.py mode change 100755 => 100644 airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/client.py diff --git a/airbyte-integrations/bases/base-python/base_python/__init__.py b/airbyte-integrations/bases/base-python/base_python/__init__.py old mode 100755 new mode 100644 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/client.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/client/client.py old mode 100755 new mode 100644 From d2755bdabbf3e091023fd04fd0b106fb4c5c3cb8 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Thu, 3 Jun 2021 17:09:37 +0300 Subject: [PATCH 05/19] chmod on files changed --- .../connectors/source-googleanalytics-singer/main_dev.py | 0 .../source_googleanalytics_singer/source.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 airbyte-integrations/connectors/source-googleanalytics-singer/main_dev.py mode change 100755 => 100644 airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/main_dev.py b/airbyte-integrations/connectors/source-googleanalytics-singer/main_dev.py old mode 100755 new mode 100644 diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py old mode 100755 new mode 100644 From 5d5ca42d2600c8fe0486ab5dff6729c76d8a0d45 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Thu, 3 Jun 2021 17:11:59 +0300 Subject: [PATCH 06/19] chmod on files changed --- .../connectors/source-smartsheets/source_smartsheets/source.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 airbyte-integrations/connectors/source-smartsheets/source_smartsheets/source.py diff --git a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/source.py b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/source.py old mode 100755 new mode 100644 From 6fed47d0686a63795957812bea29c53bc4557f1f Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Tue, 8 Jun 2021 11:08:29 +0300 Subject: [PATCH 07/19] added acceptance tests --- .../acceptance-test-config.yml | 26 + .../build.gradle | 10 +- .../integration_tests/__init__.py | 0 .../integration_tests/acceptance.py | 36 ++ .../integration_tests/configured_catalog.json | 523 ++++++++++++++++++ .../integration_tests/invalid_config.json | 6 + .../sample_files/abnormal_state.json | 5 + 7 files changed, 598 insertions(+), 8 deletions(-) create mode 100644 airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml b/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml new file mode 100644 index 000000000000..5acdf27946d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml @@ -0,0 +1,26 @@ +connector_image: airbyte/source-googleanalytics-singer:dev +tests: + spec: + - spec_path: "source_googleanalytics_singer/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + state_path: "sample_files/abnormal_state.json" + cursor_paths: + website_overview: ["report_start_date"] + traffic_sources: ["report_start_date"] + pages: ["report_start_date"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle b/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle index 446aef2c59df..8f4041949abd 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle @@ -1,7 +1,7 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-standard-source-test-file' + id 'airbyte-source-acceptance-test' } airbytePython { @@ -10,12 +10,6 @@ airbytePython { dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) -} - -airbyteStandardSourceTestFile { - specPath = "source_googleanalytics_singer/spec.json" - configPath = "secrets/config.json" - configuredCatalogPath = "sample_files/test_catalog.json" + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/acceptance.py new file mode 100644 index 000000000000..d98ac8aa3a1c --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..9381437e5c2b --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/configured_catalog.json @@ -0,0 +1,523 @@ +{ + "streams": [ + { + "stream": { + "name": "website_overview", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_pageviewsPerSession": { + "type": ["null", "number"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "traffic_sources", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_source": { + "type": ["string"] + }, + "ga_medium": { + "type": ["string"] + }, + "ga_socialNetwork": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_pageviewsPerSession": { + "type": ["null", "number"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "pages", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_hostname": { + "type": ["string"] + }, + "ga_pagePath": { + "type": ["string"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_uniquePageviews": { + "type": ["null", "integer"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_entrances": { + "type": ["null", "integer"] + }, + "ga_entranceRate": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exits": { + "type": ["null", "integer"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "locations", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_continent": { + "type": ["string"] + }, + "ga_subContinent": { + "type": ["string"] + }, + "ga_country": { + "type": ["string"] + }, + "ga_region": { + "type": ["string"] + }, + "ga_metro": { + "type": ["string"] + }, + "ga_city": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_pageviewsPerSession": { + "type": ["null", "number"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "monthly_active_users", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_30dayUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "four_weekly_active_users", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_28dayUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "two_weekly_active_users", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_14dayUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "weekly_active_users", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_7dayUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "daily_active_users", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_1dayUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "devices", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_deviceCategory": { + "type": ["string"] + }, + "ga_operatingSystem": { + "type": ["string"] + }, + "ga_browser": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_pageviewsPerSession": { + "type": ["null", "number"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "users_per_day", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "sessions_per_country_day", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_country": { + "type": ["string"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json new file mode 100644 index 000000000000..397b5b03d41f --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "credentials_json": "{\n \"type\": \"service_account\",\n \"project_id\": \"dataline-integration-testing\",\n \"private_key_id\": \"8e24553ba1c810d663a74dc5e0b4463911ae3dbb\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAooBAQDp7M8lWSO2fn/3\\nriGVX5GnTNqB4rGHs5IhKtoOAQ+/Vt+4hS9+VxyMyx8+k9/I/3KJRd5gOTA3bD45\\nXyqvLecOoyAVKyHtdtePmm8zjQgtAS9Hx0zuxDZA605qh5IR76j564wgdZtZfF+o\\nvnK6HqM0pCJQuYNnZmetGuOh+1vE7asF2SUkLo3qTVsnEreQdjV6M/aeUehtsJK0\\nnSZ2VAoXAgS4i+7vmrxCxZ9iRnzKSBaBXDthAFA4kap1DWMpnWI+GH00vSF3Ofb9\\nYyvArv9Fqj66uDqD0fMZJvji4+USQTziXGyZ8aO+7bJ5c3aR+qytkQmd52fS37U9\\nJlPeUBOzAgMBAAECggEAAJ4UaefTVU7+LXwzMH7BFqz4QrGDgHUz5IyFaNYN4ZZs\\nHsOTPYUjWSfMHFEtnQ03ky8xCBX+j5bkjyg2J6I6YuqBUiz7+PGAXiutMXik1eGT\\n61+WT2cLMZ6tZVCYTR3R118MzqUg1NqkMgDWsrEUAzaPtKDYvPQjP+y7Pxk3w8xP\\nhS/EIEN5EWczWencfouYfx11ZC6I9IwkywQR3kKxWhaaxOVUr3LYTkW9r5g7dTgE\\n0u+NCJRTJ7uZmQ0pSWcAkWV/42jEY8j/PLAdzpEFO0wf/9pPBAJrtSvEAH8qWx6H\\ne5sjAcqW0S3d0CMVstAUjlJcGGiQNfYB6oWql3+tgQKBgQD/6Hp97W1F6kAAiRCH\\nt5ES0JynbwCGvtoExPnip5RNHQuxIs+FI7xNSsPUJii7nIOIpjBog5coFOhRF7fp\\n5oraALS6d2AQRDfTn8R1XPsfX6WWElpp7eSuU7xLB0yHZed/iP7SRsArRxeTVNHj\\nRIh6zPM0avAZvOBGVk4CjunocwKBgQDqAk9klJLq9UveJdUwua/NQ42ubBjylA5I\\nJpeZJTmnGpTfbCeTjmnlLTWlvnT7s7inFFPSrAxS96F1/vwv61alPsUBBmHNytom\\nypmKdzH0JR7bALJbNpNr2W7c0ncHbOPya6+QEq5ZvF/AbsVVTcX++GrnTKCP5lJ7\\nb5n/rbSXwQKBgQDaKxhCw2elc0+dJT0ydwz3PWJQXBHWzl6QMl3XHYcRNvIA0eyZ\\nVR3zxwmsk9umFokMtqIOeCElyq1EdTQV9LXrpS5uydbcB0yQ3mReqiZtWN2SU5NB\\nO875z+l3DYHw7K+vyttz6V1rh/BRar6FQqgYCSJOTdzkFGIflPRsueowgQKBgQDn\\nP7ktcBVv01BSC5kOObGdavCMOY69ycoDSv/s+3hpxj7wO3UwwD+tlmu5iukYA5aq\\npc/gFN1o8AXDEWQuBqbtTYZpaOEl+Rxz0SrCRuA6oKRJT/rxYJF2trxzxBiUYesx\\nGXG5MnzRePI2vECN9/l96gfa40KYccd46+SHYiVhAQKBgQCK66RqjCFxJSoeD++B\\nGnIG3hFM2wJg0BsYlJPwg82OF/efhPmPMzbFtmZw4BbVZ8KGSPAJ3oa7HskKKsmD\\nlnsrNtbRQ8EGXpBRr6d3cQhoShIiicqh0b2jvZgPHKOlgexMi6A3G+rk6v3PlyaL\\nXlnklGqG4MilnFgyKs5nEah1ow==\\n-----END PRIVATE KEY-----\\n\",\n \"client_email\": \"google-analytics-access@dataline-integration-testing.iam.gserviceaccount.com\",\n \"client_id\": \"103293192021796062643\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/google-analytics-access%40dataline-integration-testing.iam.gserviceaccount.com\"\n}\n", + "view_id": "222222222", + "start_date": "2020-02-13T00:00:00Z", + "custom_reports": "[{\"name\": \"users_per_day\", \"dimensions\": [\"ga:date\"], \"metrics\": [\"ga:users\", \"ga:newUsers\"]}, {\"name\": \"sessions_per_country_day\", \"dimensions\": [\"ga:date\", \"ga:country\"], \"metrics\": [\"ga:sessions\", \"ga:sessionsPerUser\", \"ga:avgSessionDuration\"]}]" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json b/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json new file mode 100644 index 000000000000..28ce1ab4f190 --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "website_overview": {"report_start_date": "2050-05-01T00:00:00Z"}, + "traffic_sources": {"report_start_date": "2050-05-01T00:00:00Z"}, + "pages": {"report_start_date": "2050-05-01T00:00:00Z"} +} \ No newline at end of file From 52533877343d1b15453208da580573c6b233051b Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Wed, 9 Jun 2021 11:58:17 +0300 Subject: [PATCH 08/19] added pre commit config --- .pre-commit-config.yaml | 21 + .../sample_files/configured_catalog.json | 400 ------------------ 2 files changed, 21 insertions(+), 400 deletions(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000000..395cd9b5b172 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +repos: + - repo: https://github.com/ambv/black + rev: stable + hooks: + - id: black + args: ["--line-length=140"] + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v1.2.3 + hooks: + - id: flake8 + args: ["--config=tools/python/.flake8"] + - repo: https://github.com/timothycrosley/isort + rev: 4.3.21 + hooks: + - id: isort + args: ["--settings-path=tools/python/.isort.cfg"] + - repo: https://github.com/johann-petrak/licenseheaders.git + rev: 'master' + hooks: + - id: licenseheaders + args: ["--tmpl=LICENSE", "--ext=py", "-f"] diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/configured_catalog.json index 9381437e5c2b..6cfc5158b0e4 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/configured_catalog.json @@ -118,406 +118,6 @@ "sync_mode": "incremental", "cursor_field": ["report_start_date"], "destination_sync_mode": "append" - }, - { - "stream": { - "name": "pages", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_hostname": { - "type": ["string"] - }, - "ga_pagePath": { - "type": ["string"] - }, - "ga_pageviews": { - "type": ["null", "integer"] - }, - "ga_uniquePageviews": { - "type": ["null", "integer"] - }, - "ga_avgTimeOnPage": { - "type": ["null", "number"] - }, - "ga_entrances": { - "type": ["null", "integer"] - }, - "ga_entranceRate": { - "type": ["null", "number"] - }, - "ga_bounceRate": { - "type": ["null", "number"] - }, - "ga_exits": { - "type": ["null", "integer"] - }, - "ga_exitRate": { - "type": ["null", "number"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "locations", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_continent": { - "type": ["string"] - }, - "ga_subContinent": { - "type": ["string"] - }, - "ga_country": { - "type": ["string"] - }, - "ga_region": { - "type": ["string"] - }, - "ga_metro": { - "type": ["string"] - }, - "ga_city": { - "type": ["string"] - }, - "ga_users": { - "type": ["null", "integer"] - }, - "ga_newUsers": { - "type": ["null", "integer"] - }, - "ga_sessions": { - "type": ["null", "integer"] - }, - "ga_sessionsPerUser": { - "type": ["null", "number"] - }, - "ga_avgSessionDuration": { - "type": ["null", "number"] - }, - "ga_pageviews": { - "type": ["null", "integer"] - }, - "ga_pageviewsPerSession": { - "type": ["null", "number"] - }, - "ga_avgTimeOnPage": { - "type": ["null", "number"] - }, - "ga_bounceRate": { - "type": ["null", "number"] - }, - "ga_exitRate": { - "type": ["null", "number"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "monthly_active_users", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_30dayUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "four_weekly_active_users", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_28dayUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "two_weekly_active_users", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_14dayUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "weekly_active_users", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_7dayUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "daily_active_users", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_1dayUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "devices", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_deviceCategory": { - "type": ["string"] - }, - "ga_operatingSystem": { - "type": ["string"] - }, - "ga_browser": { - "type": ["string"] - }, - "ga_users": { - "type": ["null", "integer"] - }, - "ga_newUsers": { - "type": ["null", "integer"] - }, - "ga_sessions": { - "type": ["null", "integer"] - }, - "ga_sessionsPerUser": { - "type": ["null", "number"] - }, - "ga_avgSessionDuration": { - "type": ["null", "number"] - }, - "ga_pageviews": { - "type": ["null", "integer"] - }, - "ga_pageviewsPerSession": { - "type": ["null", "number"] - }, - "ga_avgTimeOnPage": { - "type": ["null", "number"] - }, - "ga_bounceRate": { - "type": ["null", "number"] - }, - "ga_exitRate": { - "type": ["null", "number"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "users_per_day", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_users": { - "type": ["null", "integer"] - }, - "ga_newUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "sessions_per_country_day", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_country": { - "type": ["string"] - }, - "ga_sessions": { - "type": ["null", "integer"] - }, - "ga_sessionsPerUser": { - "type": ["null", "number"] - }, - "ga_avgSessionDuration": { - "type": ["null", "number"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" } ] } From e77c90b583eedd7fdee85a4db62da6255e89ce26 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Wed, 9 Jun 2021 21:43:43 +0300 Subject: [PATCH 09/19] fixed chmod on amazon and facebook files --- .../api/generated-api-html/index.html | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 67f69e815283..424441e3be81 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -412,6 +412,7 @@

Example data

} ] }, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "namespaceFormat" : "${SOURCE_NAMESPACE}", "operationIds" : [ null, null ], "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } @@ -551,6 +552,7 @@

Example data

} ] }, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "namespaceFormat" : "${SOURCE_NAMESPACE}", "operationIds" : [ null, null ], "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } @@ -710,6 +712,7 @@

Example data

} ] }, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "namespaceFormat" : "${SOURCE_NAMESPACE}", "operationIds" : [ null, null ], "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" }, { @@ -754,6 +757,7 @@

Example data

} ] }, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "namespaceFormat" : "${SOURCE_NAMESPACE}", "operationIds" : [ null, null ], "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } ] @@ -1031,6 +1035,7 @@

Example data

} ] }, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "namespaceFormat" : "${SOURCE_NAMESPACE}", "operationIds" : [ null, null ], "destinationId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91" } @@ -4097,6 +4102,7 @@

Example data

} ] }, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "namespaceFormat" : "${SOURCE_NAMESPACE}", "operationIds" : [ null, null ] } @@ -4247,6 +4253,7 @@

Example data

} ] }, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "namespaceFormat" : "${SOURCE_NAMESPACE}", "operationIds" : [ null, null ] } @@ -4401,6 +4408,7 @@

Example data

} ] }, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "namespaceFormat" : "${SOURCE_NAMESPACE}", "operationIds" : [ null, null ] }, { "sourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", @@ -4498,6 +4506,7 @@

Example data

} ] }, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "namespaceFormat" : "${SOURCE_NAMESPACE}", "operationIds" : [ null, null ] } ] } @@ -4776,6 +4785,7 @@

Example data

} ] }, "connectionId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91", + "namespaceFormat" : "${SOURCE_NAMESPACE}", "operationIds" : [ null, null ] } @@ -5190,6 +5200,7 @@

Table of Contents

  • LogRead -
  • LogType -
  • LogsRequestBody -
  • +
  • NamespaceDefinitionType -
  • Notification -
  • NotificationRead -
  • NotificationType -
  • @@ -5328,6 +5339,8 @@

    ConnectionCreate -
    name (optional)
    String Optional name of the connection
    +
    namespaceDefinition (optional)
    +
    namespaceFormat (optional)
    String Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'.
    prefix (optional)
    String Prefix that will be prepended to the name of each stream when it is written to the destination.
    sourceId
    UUID format: uuid
    destinationId
    UUID format: uuid
    @@ -5350,6 +5363,8 @@

    ConnectionRead -
    connectionId
    UUID format: uuid
    name
    +
    namespaceDefinition (optional)
    +
    namespaceFormat (optional)
    String Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'.
    prefix (optional)
    String Prefix that will be prepended to the name of each stream when it is written to the destination.
    sourceId
    UUID format: uuid
    destinationId
    UUID format: uuid
    @@ -5395,6 +5410,8 @@

    ConnectionUpdate -
    connectionId
    UUID format: uuid
    +
    namespaceDefinition (optional)
    +
    namespaceFormat (optional)
    String Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'.
    prefix (optional)
    String Prefix that will be prepended to the name of each stream when it is written to the destination.
    operationIds (optional)
    array[UUID] format: uuid
    syncCatalog
    @@ -5633,6 +5650,12 @@

    LogsRequestBody - logType +
    +

    NamespaceDefinitionType - Up

    +
    Method used for computing final namespace in destination
    +
    +
    +

    Notification - Up

    @@ -5913,6 +5936,8 @@

    WebBackendConnectionCreate
    name (optional)
    String Optional name of the connection
    +
    namespaceDefinition (optional)
    +
    namespaceFormat (optional)
    String Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'.
    prefix (optional)
    String Prefix that will be prepended to the name of each stream when it is written to the destination.
    sourceId
    UUID format: uuid
    destinationId
    UUID format: uuid
    @@ -5936,7 +5961,9 @@

    WebBackendConnectionRead - <
    connectionId
    UUID format: uuid
    name
    -
    prefix (optional)
    +
    namespaceDefinition (optional)
    +
    namespaceFormat (optional)
    String Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'.
    +
    prefix (optional)
    String Prefix that will be prepended to the name of each stream when it is written to the destination.
    sourceId
    UUID format: uuid
    destinationId
    UUID format: uuid
    syncCatalog
    @@ -5971,6 +5998,8 @@

    WebBackendConnectionUpdate
    connectionId
    UUID format: uuid
    +
    namespaceDefinition (optional)
    +
    namespaceFormat (optional)
    String Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'.
    prefix (optional)
    String Prefix that will be prepended to the name of each stream when it is written to the destination.
    operationIds (optional)
    array[UUID] format: uuid
    syncCatalog
    From f5ae757235bc375086e3c9b6d9be7d6d6b8d2283 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Wed, 9 Jun 2021 21:43:57 +0300 Subject: [PATCH 10/19] fixed chmod on amazon and facebook files --- .../destination-java/.dockerignore.hbs | 3 + .../destination-java/Destination.java.hbs | 61 + .../DestinationAcceptanceTest.java.hbs | 78 + .../destination-java/Dockerfile.hbs | 11 + .../destination-java/README.md.hbs | 73 + .../destination-java/build.gradle.hbs | 19 + .../destination-java/definition.yaml.hbs | 5 + .../destination-definition.json.hbs | 7 + .../destination-java/doc.md.hbs | 52 + .../destination-java/spec.json.hbs | 22 + .../source-java-jdbc/.dockerignore | 3 + .../source-java-jdbc/Dockerfile | 13 + .../source-java-jdbc/build.gradle | 28 + .../{{pascalCase name}}Source.java.hbs | 70 + .../src/main/resources/spec.json.hbs | 55 + ...alCase name}}SourceAcceptanceTest.java.hbs | 87 + ...se name}}JdbcSourceAcceptanceTest.java.hbs | 88 + .../{{pascalCase name}}SourceTests.java.hbs | 51 + .../destination-oracle/.dockerignore | 3 + .../connectors/destination-oracle/Dockerfile | 12 + .../destination-oracle/build.gradle | 31 + .../destination/oracle/OracleDestination.java | 75 + .../oracle/OracleNameTransformer.java | 65 + .../destination/oracle/OracleOperations.java | 184 + .../src/main/resources/spec.json | 57 + .../oracle/OracleIntegrationTest.java | 210 + .../connectors/destination-s3/.dockerignore | 3 + .../connectors/destination-s3/Dockerfile | 11 + .../connectors/destination-s3/build.gradle | 26 + .../destination/s3/S3Consumer.java | 125 + .../destination/s3/S3Destination.java | 72 + .../destination/s3/S3DestinationConfig.java | 87 + .../s3/S3DestinationConstants.java | 45 + .../integrations/destination/s3/S3Format.java | 29 + .../destination/s3/S3FormatConfig.java | 31 + .../destination/s3/S3FormatConfigs.java | 46 + .../destination/s3/S3OutputFormatter.java | 52 + .../s3/S3OutputFormatterFactory.java | 43 + .../S3OutputFormatterProductionFactory.java | 49 + .../s3/csv/BaseSheetGenerator.java | 49 + .../destination/s3/csv/CsvSheetGenerator.java | 58 + .../s3/csv/CsvSheetGenerators.java | 29 + .../s3/csv/NoFlatteningSheetGenerator.java | 52 + .../RootLevelFlatteningSheetGenerator.java | 81 + .../destination/s3/csv/S3CsvFormatConfig.java | 77 + .../s3/csv/S3CsvOutputFormatter.java | 192 + .../src/main/resources/spec.json | 103 + .../s3/S3CsvDestinationAcceptanceTest.java | 243 ++ .../destination/s3/S3FormatConfigsTest.java | 58 + .../csv/NoFlatteningSheetGeneratorTest.java | 64 + ...RootLevelFlatteningSheetGeneratorTest.java | 87 + .../s3/csv/S3CsvFormatConfigTest.java | 48 + .../s3/csv/S3CsvOutputFormatterTest.java | 59 + .../connectors/source-amplitude/.dockerignore | 6 + .../connectors/source-amplitude/CHANGELOG.md | 4 + .../connectors/source-amplitude/Dockerfile | 15 + .../connectors/source-amplitude/README.md | 129 + .../acceptance-test-config.yml | 25 + .../acceptance-test-docker.sh | 7 + .../connectors/source-amplitude/build.gradle | 13 + .../integration_tests/__init__.py | 0 .../integration_tests/acceptance.py | 36 + .../integration_tests/configured_catalog.json | 317 ++ .../configured_catalog_without_events.json | 145 + .../integration_tests/invalid_config.json | 5 + .../integration_tests/sample_config.json | 5 + .../integration_tests/sample_state.json | 11 + .../streams_with_output_records_catalog.json | 117 + .../connectors/source-amplitude/main.py | 33 + .../connectors/source-amplitude/setup.py | 45 + .../source_amplitude/__init__.py | 27 + .../source-amplitude/source_amplitude/api.py | 209 + .../schemas/active_users.json | 13 + .../source_amplitude/schemas/annotations.json | 19 + .../schemas/average_session_length.json | 13 + .../source_amplitude/schemas/cohorts.json | 54 + .../source_amplitude/schemas/events.json | 160 + .../source_amplitude/source.py | 64 + .../source_amplitude/spec.json | 28 + .../source-amplitude/unit_tests/unit_test.py | 27 + .../connectors/source-harvest/.dockerignore | 5 + .../connectors/source-harvest/Dockerfile | 15 + .../connectors/source-harvest/README.md | 129 + .../source-harvest/acceptance-test-config.yml | 25 + .../source-harvest/acceptance-test-docker.sh | 7 + .../connectors/source-harvest/build.gradle | 13 + .../integration_tests/__init__.py | 0 .../integration_tests/abnormal_state.json | 86 + .../integration_tests/acceptance.py | 36 + .../integration_tests/configured_catalog.json | 2060 ++++++++++ .../integration_tests/invalid_config.json | 5 + .../integration_tests/sample_config.json | 5 + .../integration_tests/sample_state.json | 86 + .../connectors/source-harvest/main.py | 33 + .../connectors/source-harvest/setup.py | 47 + .../source-harvest/source_harvest/__init__.py | 27 + .../source-harvest/source_harvest/auth.py | 37 + .../schemas/billable_rates.json | 28 + .../source_harvest/schemas/clients.json | 33 + .../source_harvest/schemas/company.json | 57 + .../source_harvest/schemas/contacts.json | 49 + .../source_harvest/schemas/cost_rates.json | 28 + .../schemas/estimate_item_categories.json | 20 + .../schemas/estimate_messages.json | 44 + .../source_harvest/schemas/estimates.json | 98 + .../schemas/expense_categories.json | 29 + .../source_harvest/schemas/expenses.json | 157 + .../schemas/expenses_categories.json | 27 + .../schemas/expenses_clients.json | 27 + .../schemas/expenses_projects.json | 33 + .../source_harvest/schemas/expenses_team.json | 30 + .../schemas/invoice_item_categories.json | 26 + .../schemas/invoice_messages.json | 67 + .../schemas/invoice_payments.json | 51 + .../source_harvest/schemas/invoices.json | 141 + .../schemas/project_assignments.json | 60 + .../schemas/project_budget.json | 42 + .../source_harvest/schemas/projects.json | 93 + .../source_harvest/schemas/roles.json | 23 + .../schemas/task_assignments.json | 54 + .../source_harvest/schemas/tasks.json | 32 + .../source_harvest/schemas/time_clients.json | 30 + .../source_harvest/schemas/time_entries.json | 174 + .../source_harvest/schemas/time_projects.json | 36 + .../source_harvest/schemas/time_tasks.json | 30 + .../source_harvest/schemas/time_team.json | 33 + .../source_harvest/schemas/uninvoiced.json | 39 + .../schemas/user_assignments.json | 57 + .../source_harvest/schemas/users.json | 71 + .../source-harvest/source_harvest/source.py | 127 + .../source-harvest/source_harvest/spec.json | 33 + .../source-harvest/source_harvest/streams.py | 457 +++ .../source-harvest/unit_tests/unit_test.py | 27 + .../connectors/source-posthog/.dockerignore | 5 + .../connectors/source-posthog/CHANGELOG.md | 4 + .../connectors/source-posthog/Dockerfile | 15 + .../connectors/source-posthog/README.md | 129 + .../source-posthog/acceptance-test-config.yml | 22 + .../source-posthog/acceptance-test-docker.sh | 8 + .../connectors/source-posthog/build.gradle | 14 + .../integration_tests/__init__.py | 0 .../integration_tests/acceptance.py | 36 + .../integration_tests/configured_catalog.json | 139 + .../integration_tests/invalid_config.json | 4 + .../integration_tests/state.json | 6 + .../connectors/source-posthog/main.py | 33 + .../source-posthog/requirements.txt | 2 + .../source-posthog/sample_files/state.json | 6 + .../connectors/source-posthog/setup.py | 46 + .../source-posthog/source_posthog/__init__.py | 3 + .../source_posthog/schemas/annotations.json | 55 + .../source_posthog/schemas/cohorts.json | 73 + .../source_posthog/schemas/events.json | 44 + .../schemas/events_sessions.json | 196 + .../source_posthog/schemas/feature_flags.json | 55 + .../source_posthog/schemas/insights.json | 124 + .../source_posthog/schemas/insights_path.json | 20 + .../schemas/insights_sessions.json | 32 + .../source_posthog/schemas/persons.json | 24 + .../source_posthog/schemas/sessions.json | 158 + .../source_posthog/schemas/trends.json | 58 + .../source-posthog/source_posthog/source.py | 81 + .../source-posthog/source_posthog/spec.json | 24 + .../source-posthog/source_posthog/streams.py | 230 ++ .../source-posthog/unit_tests/unit_test.py | 33 + .../source-scaffold-java-jdbc/Dockerfile | 13 + .../source-scaffold-java-jdbc/build.gradle | 28 + .../ScaffoldJavaJdbcSource.java | 70 + .../src/main/resources/spec.json | 55 + .../ScaffoldJavaJdbcSourceAcceptanceTest.java | 87 + ...ffoldJavaJdbcJdbcSourceAcceptanceTest.java | 88 + .../ScaffoldJavaJdbcSourceTests.java | 51 + .../connectors/source-shopify/.dockerignore | 7 + .../connectors/source-shopify/CHANGELOG.md | 7 + .../connectors/source-shopify/Dockerfile | 15 + .../connectors/source-shopify/README.md | 139 + .../source-shopify/acceptance-test-config.yml | 25 + .../source-shopify/acceptance-test-docker.sh | 7 + .../connectors/source-shopify/build.gradle | 9 + .../integration_tests/__init__.py | 0 .../integration_tests/abnormal_state.json | 32 + .../integration_tests/acceptance.py | 36 + .../integration_tests/configured_catalog.json | 3406 +++++++++++++++++ .../integration_tests/invalid_config.json | 5 + .../integration_tests/no_refunds_catalog.json | 3109 +++++++++++++++ .../integration_tests/state.json | 32 + .../connectors/source-shopify/main.py | 33 + .../source-shopify/requirements.txt | 2 + .../connectors/source-shopify/setup.py | 48 + .../source-shopify/source_shopify/__init__.py | 27 + .../source_shopify/schemas/TODO.md | 25 + .../schemas/abandoned_checkouts.json | 770 ++++ .../source_shopify/schemas/collects.json | 28 + .../schemas/custom_collections.json | 55 + .../source_shopify/schemas/customers.json | 196 + .../source_shopify/schemas/metafields.json | 41 + .../source_shopify/schemas/order_refunds.json | 395 ++ .../source_shopify/schemas/order_risks.json | 35 + .../source_shopify/schemas/orders.json | 1426 +++++++ .../source_shopify/schemas/products.json | 1581 ++++++++ .../source_shopify/schemas/transactions.json | 100 + .../source-shopify/source_shopify/source.py | 263 ++ .../source-shopify/source_shopify/spec.json | 27 + .../source-shopify/unit_tests/unit_test.py | 27 + .../NamespaceDefinitionType.yaml | 11 + .../migrationV0_25_0/StandardSync.yaml | 70 + docs/troubleshooting/README.md | 2 + docs/troubleshooting/new-connection.md | 34 + docs/troubleshooting/on-deploy.md | 69 + docs/troubleshooting/running-sync.md | 33 + docs/troubleshooting/upgrading.md | 0 211 files changed, 23886 insertions(+) create mode 100644 airbyte-integrations/connector-templates/destination-java/.dockerignore.hbs create mode 100644 airbyte-integrations/connector-templates/destination-java/Destination.java.hbs create mode 100644 airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs create mode 100644 airbyte-integrations/connector-templates/destination-java/Dockerfile.hbs create mode 100644 airbyte-integrations/connector-templates/destination-java/README.md.hbs create mode 100644 airbyte-integrations/connector-templates/destination-java/build.gradle.hbs create mode 100644 airbyte-integrations/connector-templates/destination-java/definition.yaml.hbs create mode 100644 airbyte-integrations/connector-templates/destination-java/destination-definition.json.hbs create mode 100644 airbyte-integrations/connector-templates/destination-java/doc.md.hbs create mode 100644 airbyte-integrations/connector-templates/destination-java/spec.json.hbs create mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore create mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/Dockerfile create mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/build.gradle create mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs create mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs create mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs create mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs create mode 100644 airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs create mode 100644 airbyte-integrations/connectors/destination-oracle/.dockerignore create mode 100644 airbyte-integrations/connectors/destination-oracle/Dockerfile create mode 100644 airbyte-integrations/connectors/destination-oracle/build.gradle create mode 100644 airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java create mode 100644 airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java create mode 100644 airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java create mode 100644 airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json create mode 100644 airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleIntegrationTest.java create mode 100644 airbyte-integrations/connectors/destination-s3/.dockerignore create mode 100644 airbyte-integrations/connectors/destination-s3/Dockerfile create mode 100644 airbyte-integrations/connectors/destination-s3/build.gradle create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Consumer.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Format.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatter.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterFactory.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterProductionFactory.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerators.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatter.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json create mode 100644 airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigsTest.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGeneratorTest.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGeneratorTest.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatterTest.java create mode 100644 airbyte-integrations/connectors/source-amplitude/.dockerignore create mode 100644 airbyte-integrations/connectors/source-amplitude/CHANGELOG.md create mode 100644 airbyte-integrations/connectors/source-amplitude/Dockerfile create mode 100644 airbyte-integrations/connectors/source-amplitude/README.md create mode 100644 airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-amplitude/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-amplitude/build.gradle create mode 100644 airbyte-integrations/connectors/source-amplitude/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog_without_events.json create mode 100644 airbyte-integrations/connectors/source-amplitude/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-amplitude/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-amplitude/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-amplitude/integration_tests/streams_with_output_records_catalog.json create mode 100644 airbyte-integrations/connectors/source-amplitude/main.py create mode 100644 airbyte-integrations/connectors/source-amplitude/setup.py create mode 100644 airbyte-integrations/connectors/source-amplitude/source_amplitude/__init__.py create mode 100644 airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py create mode 100644 airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/active_users.json create mode 100644 airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/annotations.json create mode 100644 airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json create mode 100644 airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/cohorts.json create mode 100644 airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/events.json create mode 100644 airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py create mode 100644 airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.json create mode 100644 airbyte-integrations/connectors/source-amplitude/unit_tests/unit_test.py create mode 100644 airbyte-integrations/connectors/source-harvest/.dockerignore create mode 100644 airbyte-integrations/connectors/source-harvest/Dockerfile create mode 100644 airbyte-integrations/connectors/source-harvest/README.md create mode 100644 airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml create mode 100755 airbyte-integrations/connectors/source-harvest/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-harvest/build.gradle create mode 100644 airbyte-integrations/connectors/source-harvest/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-harvest/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-harvest/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-harvest/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-harvest/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-harvest/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-harvest/main.py create mode 100644 airbyte-integrations/connectors/source-harvest/setup.py create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/__init__.py create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/auth.py create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/billable_rates.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/clients.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/company.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/contacts.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/cost_rates.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_item_categories.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_messages.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimates.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expense_categories.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_categories.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_clients.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_projects.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_team.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_item_categories.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_messages.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_payments.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_assignments.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_budget.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/projects.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/roles.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/task_assignments.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/tasks.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_clients.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_entries.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_projects.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_tasks.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_team.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/uninvoiced.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/user_assignments.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/schemas/users.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/source.py create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/spec.json create mode 100644 airbyte-integrations/connectors/source-harvest/source_harvest/streams.py create mode 100644 airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py create mode 100644 airbyte-integrations/connectors/source-posthog/.dockerignore create mode 100644 airbyte-integrations/connectors/source-posthog/CHANGELOG.md create mode 100644 airbyte-integrations/connectors/source-posthog/Dockerfile create mode 100644 airbyte-integrations/connectors/source-posthog/README.md create mode 100644 airbyte-integrations/connectors/source-posthog/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-posthog/build.gradle create mode 100644 airbyte-integrations/connectors/source-posthog/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-posthog/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-posthog/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-posthog/integration_tests/state.json create mode 100644 airbyte-integrations/connectors/source-posthog/main.py create mode 100644 airbyte-integrations/connectors/source-posthog/requirements.txt create mode 100644 airbyte-integrations/connectors/source-posthog/sample_files/state.json create mode 100644 airbyte-integrations/connectors/source-posthog/setup.py create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/__init__.py create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/annotations.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/cohorts.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events_sessions.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/feature_flags.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_path.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_sessions.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/persons.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/sessions.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/schemas/trends.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/source.py create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/spec.json create mode 100644 airbyte-integrations/connectors/source-posthog/source_posthog/streams.py create mode 100644 airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py create mode 100644 airbyte-integrations/connectors/source-scaffold-java-jdbc/Dockerfile create mode 100644 airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle create mode 100644 airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java create mode 100644 airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json create mode 100644 airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcJdbcSourceAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceTests.java create mode 100644 airbyte-integrations/connectors/source-shopify/.dockerignore create mode 100644 airbyte-integrations/connectors/source-shopify/CHANGELOG.md create mode 100644 airbyte-integrations/connectors/source-shopify/Dockerfile create mode 100644 airbyte-integrations/connectors/source-shopify/README.md create mode 100644 airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-shopify/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-shopify/build.gradle create mode 100644 airbyte-integrations/connectors/source-shopify/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-shopify/integration_tests/no_refunds_catalog.json create mode 100644 airbyte-integrations/connectors/source-shopify/integration_tests/state.json create mode 100644 airbyte-integrations/connectors/source-shopify/main.py create mode 100644 airbyte-integrations/connectors/source-shopify/requirements.txt create mode 100644 airbyte-integrations/connectors/source-shopify/setup.py create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/__init__.py create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collects.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafields.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/source.py create mode 100644 airbyte-integrations/connectors/source-shopify/source_shopify/spec.json create mode 100644 airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py create mode 100644 airbyte-migration/src/main/resources/migrations/migrationV0_25_0/NamespaceDefinitionType.yaml create mode 100644 airbyte-migration/src/main/resources/migrations/migrationV0_25_0/StandardSync.yaml create mode 100644 docs/troubleshooting/README.md create mode 100644 docs/troubleshooting/new-connection.md create mode 100644 docs/troubleshooting/on-deploy.md create mode 100644 docs/troubleshooting/running-sync.md create mode 100644 docs/troubleshooting/upgrading.md diff --git a/airbyte-integrations/connector-templates/destination-java/.dockerignore.hbs b/airbyte-integrations/connector-templates/destination-java/.dockerignore.hbs new file mode 100644 index 000000000000..65c7d0ad3e73 --- /dev/null +++ b/airbyte-integrations/connector-templates/destination-java/.dockerignore.hbs @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connector-templates/destination-java/Destination.java.hbs b/airbyte-integrations/connector-templates/destination-java/Destination.java.hbs new file mode 100644 index 000000000000..8662cc4eb1e4 --- /dev/null +++ b/airbyte-integrations/connector-templates/destination-java/Destination.java.hbs @@ -0,0 +1,61 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.{{snakeCase name}}; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.BaseConnector; +import io.airbyte.integrations.base.AirbyteMessageConsumer; +import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.protocol.models.AirbyteConnectionStatus; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class {{properCase name}}Destination extends BaseConnector implements Destination { + + private static final Logger LOGGER = LoggerFactory.getLogger({{properCase name}}Destination.class); + + public static void main(String[] args) throws Exception { + new IntegrationRunner(new {{properCase name}}Destination()).run(args); + } + + @Override + public AirbyteConnectionStatus check(JsonNode config) { + // TODO + return null; + } + + @Override + public AirbyteMessageConsumer getConsumer(JsonNode config, + ConfiguredAirbyteCatalog configuredCatalog, + Consumer outputRecordCollector) { + // TODO + return null; + } + +} diff --git a/airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs b/airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs new file mode 100644 index 000000000000..ec2da9e8c5a8 --- /dev/null +++ b/airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs @@ -0,0 +1,78 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.{{snakeCase name}}; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import java.io.IOException; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class {{properCase name}}DestinationAcceptanceTest extends DestinationAcceptanceTest { + + private static final Logger LOGGER = LoggerFactory.getLogger({{properCase name}}DestinationAcceptanceTest.class); + + private JsonNode configJson; + + @Override + protected String getImageName() { + return "airbyte/destination-{{snakeCase name}}:dev"; + } + + @Override + protected JsonNode getConfig() { + // TODO: configJson can either be static and read from secrets/config.json + // directly, or created in the setup method + return configJson; + } + + @Override + protected JsonNode getFailCheckConfig() { + // TODO + return null; + } + + @Override + protected List retrieveRecords(TestDestinationEnv testEnv, + String streamName, + String namespace, + JsonNode streamSchema) + throws IOException { + // TODO + return null; + } + + @Override + protected void setup(TestDestinationEnv testEnv) { + // TODO + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) { + // TODO + } + +} diff --git a/airbyte-integrations/connector-templates/destination-java/Dockerfile.hbs b/airbyte-integrations/connector-templates/destination-java/Dockerfile.hbs new file mode 100644 index 000000000000..5e0ba28ec0da --- /dev/null +++ b/airbyte-integrations/connector-templates/destination-java/Dockerfile.hbs @@ -0,0 +1,11 @@ +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte +ENV APPLICATION destination-{{dashCase name}} + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/destination-{{dashCase name}} diff --git a/airbyte-integrations/connector-templates/destination-java/README.md.hbs b/airbyte-integrations/connector-templates/destination-java/README.md.hbs new file mode 100644 index 000000000000..f0b90ab00544 --- /dev/null +++ b/airbyte-integrations/connector-templates/destination-java/README.md.hbs @@ -0,0 +1,73 @@ +# Destination {{titleCase name}} + +This is the repository for the {{titleCase name}} destination connector in Java. +For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/destinations/{{dashCase name}}). + +## Local development + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:build +``` + +#### Create credentials +**If you are a community contributor**, generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination{{snakeCase name}}/spec.json` file. +Note that the `secrets` directory is git-ignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, follow the [instruction](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector#using-credentials-in-ci) to set up the credentials. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/destination-{{dashCase name}}:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-{{dashCase name}}:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-{{dashCase name}}:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-{{dashCase name}}:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-{{dashCase name}}:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +We use `JUnit` for Java tests. + +### Unit and Integration Tests +Place unit and integration test files under `src/test/io/airbyte/integrations/destinations/{{snakeCase name}}`. + +#### Acceptance Tests +Airbyte has a test suite for all connectors. Place acceptance test under `src/test-integration/java/io/airbyte/integrations/destinations/{{snakeCase name}}`. + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:integrationTest +``` + +## Dependency Management + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs b/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs new file mode 100644 index 000000000000..48171415dfee --- /dev/null +++ b/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs @@ -0,0 +1,19 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.destination.{{snakeCase name}}.{{properCase name}}Destination' +} + +dependencies { + implementation project(':airbyte-config:models') + implementation project(':airbyte-protocol:models') + implementation project(':airbyte-integrations:bases:base-java') + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-{{dashCase name}}') +} diff --git a/airbyte-integrations/connector-templates/destination-java/definition.yaml.hbs b/airbyte-integrations/connector-templates/destination-java/definition.yaml.hbs new file mode 100644 index 000000000000..f318657df630 --- /dev/null +++ b/airbyte-integrations/connector-templates/destination-java/definition.yaml.hbs @@ -0,0 +1,5 @@ +- destinationDefinitionId: {{uuid}} + name: {{titleCase name}} + dockerRepository: airbyte/destination-{{dashCase name}} + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/destinations/{{dashCase name}} \ No newline at end of file diff --git a/airbyte-integrations/connector-templates/destination-java/destination-definition.json.hbs b/airbyte-integrations/connector-templates/destination-java/destination-definition.json.hbs new file mode 100644 index 000000000000..f276d687cf73 --- /dev/null +++ b/airbyte-integrations/connector-templates/destination-java/destination-definition.json.hbs @@ -0,0 +1,7 @@ +{ + "destinationDefinitionId": "{{uuid}}", + "name": "{{titleCase name}}", + "dockerRepository": "airbyte/destination-{{dashCase name}}", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/{{dashCase name}}" +} diff --git a/airbyte-integrations/connector-templates/destination-java/doc.md.hbs b/airbyte-integrations/connector-templates/destination-java/doc.md.hbs new file mode 100644 index 000000000000..f281d9f64543 --- /dev/null +++ b/airbyte-integrations/connector-templates/destination-java/doc.md.hbs @@ -0,0 +1,52 @@ +# {{titleCase name}} + +TODO: update this doc + +## Sync overview + +### Output schema + +Is the output schema fixed (e.g: for an API like Stripe)? If so, point to the connector's schema (e.g: link to Stripe’s documentation) or describe the schema here directly (e.g: include a diagram or paragraphs describing the schema). + +Describe how the connector's schema is mapped to Airbyte concepts. An example description might be: "MagicDB tables become Airbyte Streams and MagicDB columns become Airbyte Fields. In addition, an extracted\_at column is appended to each row being read." + +### Data type mapping + +This section should contain a table mapping each of the connector's data types to Airbyte types. At the moment, Airbyte uses the same types used by [JSONSchema](https://json-schema.org/understanding-json-schema/reference/index.html). `string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number` are the most commonly used data types. + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | + + +### Features + +This section should contain a table with the following format: + +| Feature | Supported?(Yes/No) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | | | +| Incremental Sync | | | +| Replicate Incremental Deletes | | | +| For databases, WAL/Logical replication | | | +| SSL connection | | | +| SSH Tunnel Support | | | +| (Any other source-specific features) | | | + +### Performance considerations + +Could this connector hurt the user's database/API/etc... or put too much strain on it in certain circumstances? For example, if there are a lot of tables or rows in a table? What is the breaking point (e.g: 100mm> records)? What can the user do to prevent this? (e.g: use a read-only replica, or schedule frequent syncs, etc..) + +## Getting started + +### Requirements + +* What versions of this connector does this implementation support? (e.g: `postgres v3.14 and above`) +* What configurations, if any, are required on the connector? (e.g: `buffer_size > 1024`) +* Network accessibility requirements +* Credentials/authentication requirements? (e.g: A DB user with read permissions on certain tables) + +### Setup guide + +For each of the above high-level requirements as appropriate, add or point to a follow-along guide. See existing source or destination guides for an example. + +For each major cloud provider we support, also add a follow-along guide for setting up Airbyte to connect to that destination. See the Postgres destination guide for an example of what this should look like. diff --git a/airbyte-integrations/connector-templates/destination-java/spec.json.hbs b/airbyte-integrations/connector-templates/destination-java/spec.json.hbs new file mode 100644 index 000000000000..c2b56ad7767f --- /dev/null +++ b/airbyte-integrations/connector-templates/destination-java/spec.json.hbs @@ -0,0 +1,22 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/{{dashCase name}}", + "supportsIncremental": TODO, + "supported_destination_sync_modes": ["TODO"], + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "TODO", + "type": "object", + "required": [ + "TODO" + ], + "additionalProperties": false, + "properties": { + "TODO_sample_field": { + "title": "Sample Field", + "type": "string", + "description": "", + "examples": [""] + } + } + } +} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore b/airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore new file mode 100644 index 000000000000..65c7d0ad3e73 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/Dockerfile b/airbyte-integrations/connector-templates/source-java-jdbc/Dockerfile new file mode 100644 index 000000000000..88d8a0591c53 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-java-jdbc/Dockerfile @@ -0,0 +1,13 @@ +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte + +ENV APPLICATION source-{{dashCase name}} + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-{{dashCase name}} \ No newline at end of file diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle b/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle new file mode 100644 index 000000000000..91bde98a2db7 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle @@ -0,0 +1,28 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.source.{{dashCase name}}.{{pascalCase name}}Source' +} + +dependencies { + implementation project(':airbyte-db') + implementation project(':airbyte-integrations:bases:base-java') + implementation project(':airbyte-protocol:models') + implementation project(':airbyte-integrations:connectors:source-jdbc') + + //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' + + testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) + + testImplementation 'org.apache.commons:commons-lang3:3.11' + + integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-{{dashCase name}}') + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') + + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) +} \ No newline at end of file diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs new file mode 100644 index 000000000000..da50488e000f --- /dev/null +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs @@ -0,0 +1,70 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.{{snakeCase name}}; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.db.jdbc.NoOpJdbcStreamingQueryConfiguration; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class {{pascalCase name}}Source extends AbstractJdbcSource implements Source { + + private static final Logger LOGGER = LoggerFactory.getLogger({{pascalCase name}}Source.class); + + // TODO insert your driver name. Ex: "com.microsoft.sqlserver.jdbc.SQLServerDriver" + static final String DRIVER_CLASS = "driver_name_here"; + + public {{pascalCase name}}Source() { + // By default NoOpJdbcStreamingQueryConfiguration class is used, but may be updated. See see example + // MssqlJdbcStreamingQueryConfiguration + super(DRIVER_CLASS, new NoOpJdbcStreamingQueryConfiguration()); + } + + // TODO The config is based on spec.json, update according to your DB + @Override + public JsonNode toJdbcConfig(JsonNode aqqConfig) { + // TODO create DB config. Ex: "Jsons.jsonNode(ImmutableMap.builder().put("username", + // userName).put("password", pas)...build()); + return null; + } + + @Override + public Set getExcludedInternalSchemas() { + // TODO Add tables to exaclude, Ex "INFORMATION_SCHEMA", "sys", "spt_fallback_db", etc + return Set.of(""); + } + + public static void main(String[] args) throws Exception { + final Source source = new {{pascalCase name}}Source(); + LOGGER.info("starting source: {}", {{pascalCase name}}Source.class); + new IntegrationRunner(source).run(args); + LOGGER.info("completed source: {}", {{pascalCase name}}Source.class); + } + +} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs new file mode 100644 index 000000000000..06ce70bee4ec --- /dev/null +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/resources/spec.json.hbs @@ -0,0 +1,55 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/source/mysql", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "{{pascalCase name}} Source Spec", + "type": "object", + "required": ["host", "port", "database", "username", "replication_method"], + "additionalProperties": false, + "properties": { + "host": { + "description": "Hostname of the database.", + "type": "string", + "order": 0 + }, + "port": { + "description": "Port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 3306, + "examples": ["3306"], + "order": 1 + }, + "database": { + "description": "Name of the database.", + "type": "string", + "order": 2 + }, + "username": { + "description": "Username to use to access the database.", + "type": "string", + "order": 3 + }, + "password": { + "description": "Password associated with the username.", + "type": "string", + "airbyte_secret": true, + "order": 4 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", + "type": "string", + "order": 5 + }, + "replication_method": { + "type": "string", + "title": "Replication Method", + "description": "Replication method to use for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "order": 6, + "default": "STANDARD", + "enum": ["STANDARD", "CDC"] + } + } + } +} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs new file mode 100644 index 000000000000..149d0435bdfb --- /dev/null +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs @@ -0,0 +1,87 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.{{snakeCase name}}; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +public class {{pascalCase name}}SourceAcceptanceTest extends SourceAcceptanceTest { + + private JsonNode config; + + @Override + protected void setup(TestDestinationEnv testEnv) throws Exception { + // TODO create new container. Ex: "new OracleContainer("epiclabs/docker-oracle-xe-11g");" + // TODO make container started. Ex: "container.start();" + // TODO init JsonNode config + // TODO crete airbyte Database object "Databases.createJdbcDatabase(...)" + // TODO insert test data to DB. Ex: "database.execute(connection-> ...)" + // TODO close Database. Ex: "database.close();" + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) { + // TODO close container that was initialized in setup() method. Ex: "container.close();" + } + + @Override + protected String getImageName() { + return "airbyte/source-{{dashCase name}}:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + // TODO Return the ConfiguredAirbyteCatalog with ConfiguredAirbyteStream objects + return null; + } + + @Override + protected List getRegexTests() { + return Collections.emptyList(); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } + +} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs new file mode 100644 index 000000000000..a0d612e22b74 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs @@ -0,0 +1,88 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.{{snakeCase name}}; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class {{pascalCase name}}JdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { + + private static final Logger LOGGER = LoggerFactory.getLogger({{pascalCase name}}JdbcSourceAcceptanceTest.class); + + // TODO declare a test container for DB. EX: org.testcontainers.containers.OracleContainer + + @BeforeAll + static void init() { + // Oracle returns uppercase values + // TODO init test container. Ex: "new OracleContainer("epiclabs/docker-oracle-xe-11g")" + // TODO start container. Ex: "container.start();" + } + + @BeforeEach + public void setup() throws Exception { + // TODO init config. Ex: "config = Jsons.jsonNode(ImmutableMap.builder().put("host", + // host).put("port", port)....build()); + super.setup(); + } + + @AfterEach + public void tearDown() { + // TODO clean used resources + } + + @Override + public AbstractJdbcSource getSource() { + return new {{pascalCase name}}Source(); + } + + @Override + public boolean supportsSchemas() { + // TODO check if your db supports it and update method accordingly + return false; + } + + @Override + public JsonNode getConfig() { + return config; + } + + @Override + public String getDriverClass() { + return {{pascalCase name}}Source.DRIVER_CLASS; + } + + @AfterAll + static void cleanUp() { + // TODO close the container. Ex: "container.close();" + } + +} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs new file mode 100644 index 000000000000..1e2f0dbe4ab8 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs @@ -0,0 +1,51 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.{{snakeCase name}}; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.db.Database; +import org.junit.jupiter.api.Test; + +public class {{pascalCase name}}SourceTests { + + private JsonNode config; + private Database database; + + @Test + public void testSettingTimezones() throws Exception { + // TODO init your container. Ex: "new + // org.testcontainers.containers.MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense();" + // TODO start the container. Ex: "container.start();" + // TODO prepare DB config. Ex: "config = getConfig(container, dbName, + // "serverTimezone=Europe/London");" + // TODO create DB, grant all privileges, etc. + // TODO check connection status. Ex: "AirbyteConnectionStatus check = new + // ScaffoldJavaJdbcGenericSource().check(config);" + // TODO assert connection status. Ex: "assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, + // check.getStatus());" + // TODO cleanup used resources and close used container. Ex: "container.close();" + } + +} diff --git a/airbyte-integrations/connectors/destination-oracle/.dockerignore b/airbyte-integrations/connectors/destination-oracle/.dockerignore new file mode 100644 index 000000000000..65c7d0ad3e73 --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile new file mode 100644 index 000000000000..09090bd8df0d --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/Dockerfile @@ -0,0 +1,12 @@ +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte + +ENV APPLICATION destination-oracle + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/destination-oracle diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle new file mode 100644 index 000000000000..35b96111cdd8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -0,0 +1,31 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.destination.oracle.OracleDestination' +} + +dependencies { + + // required so that log4j uses a standard xml parser instead of an oracle one (that gets pulled in by the oracle driver) + implementation group: 'xerces', name: 'xercesImpl', version: '2.12.1' + + implementation project(':airbyte-db') + implementation project(':airbyte-integrations:bases:base-java') + implementation project(':airbyte-protocol:models') + implementation project(':airbyte-integrations:connectors:destination-jdbc') + + implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" + + testImplementation 'org.apache.commons:commons-lang3:3.11' + testImplementation 'org.testcontainers:oracle-xe:1.15.2' + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-oracle') + + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java new file mode 100644 index 000000000000..834383f6435c --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -0,0 +1,75 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.oracle; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OracleDestination extends AbstractJdbcDestination implements Destination { + + private static final Logger LOGGER = LoggerFactory.getLogger(OracleDestination.class); + + public static final String DRIVER_CLASS = "oracle.jdbc.OracleDriver"; + + public static final String COLUMN_NAME_AB_ID = JavaBaseConstants.COLUMN_NAME_AB_ID.substring(1).toUpperCase(); + public static final String COLUMN_NAME_DATA = JavaBaseConstants.COLUMN_NAME_DATA.substring(1).toUpperCase(); + public static final String COLUMN_NAME_EMITTED_AT = JavaBaseConstants.COLUMN_NAME_EMITTED_AT.substring(1).toUpperCase(); + + public OracleDestination() { + super(DRIVER_CLASS, new OracleNameTransformer(), new OracleOperations("users")); + System.setProperty("oracle.jdbc.timezoneAsRegion", "false"); + } + + @Override + public JsonNode toJdbcConfig(JsonNode config) { + final ImmutableMap.Builder configBuilder = ImmutableMap.builder() + .put("username", config.get("username").asText()) + .put("jdbc_url", String.format("jdbc:oracle:thin:@//%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("sid").asText())); + + if (config.has("password")) { + configBuilder.put("password", config.get("password").asText()); + } + + return Jsons.jsonNode(configBuilder.build()); + } + + public static void main(String[] args) throws Exception { + final Destination destination = new OracleDestination(); + LOGGER.info("starting destination: {}", OracleDestination.class); + new IntegrationRunner(destination).run(args); + LOGGER.info("completed destination: {}", OracleDestination.class); + } + +} diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java new file mode 100644 index 000000000000..628a0161fe05 --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java @@ -0,0 +1,65 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.oracle; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.integrations.destination.ExtendedNameTransformer; +import java.util.UUID; + +@VisibleForTesting +public class OracleNameTransformer extends ExtendedNameTransformer { + + @Override + protected String applyDefaultCase(String input) { + return input.toUpperCase(); + } + + @Override + public String getRawTableName(String streamName) { + return convertStreamName("airbyte_raw_" + streamName); + } + + @Override + public String getTmpTableName(String streamName) { + return convertStreamName("airbyte_tmp_" + streamName + "_" + UUID.randomUUID().toString().replace("-", "")); + } + + private String maxStringLength(String value, Integer length) { + if (value.length() <= length) { + return value; + } + return value.substring(0, length); + } + + @Override + public String convertStreamName(String input) { + String result = super.convertStreamName(input); + if (!result.isEmpty() && result.charAt(0) == '_') { + result = result.substring(1); + } + return maxStringLength(result, 30); + } + +} diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java new file mode 100644 index 000000000000..664f0389923f --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java @@ -0,0 +1,184 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.oracle; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.destination.jdbc.SqlOperations; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.UUID; +import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class OracleOperations implements SqlOperations { + + private static final Logger LOGGER = LoggerFactory.getLogger(OracleOperations.class); + + private String tablespace; + + public OracleOperations(String tablespace) { + this.tablespace = tablespace; + } + + @Override + public void createSchemaIfNotExists(JdbcDatabase database, String schemaName) throws Exception { + if (database.queryInt("select count(*) from dba_users where upper(username) = upper(?)", schemaName) == 0) { + final String query = String.format("create user %s identified by %s quota unlimited on %s", + schemaName, schemaName, tablespace); + database.execute(query); + } + } + + @Override + public void createTableIfNotExists(JdbcDatabase database, String schemaName, String tableName) throws Exception { + try { + if (!tableExists(database, schemaName, tableName)) { + database.execute(createTableQuery(schemaName, tableName)); + } + } catch (Exception e) { + LOGGER.error("Error while creating table.", e); + throw e; + } + } + + @Override + public String createTableQuery(String schemaName, String tableName) { + return String.format( + "CREATE TABLE %s.%s ( \n" + + "%s VARCHAR(64) PRIMARY KEY,\n" + + "%s NCLOB,\n" + + "%s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP\n" + + ")", + schemaName, tableName, + OracleDestination.COLUMN_NAME_AB_ID, OracleDestination.COLUMN_NAME_DATA, OracleDestination.COLUMN_NAME_EMITTED_AT, + OracleDestination.COLUMN_NAME_DATA); + } + + private boolean tableExists(JdbcDatabase database, String schemaName, String tableName) throws Exception { + Integer count = database.queryInt("select count(*) \n from all_tables\n where upper(owner) = upper(?) and upper(table_name) = upper(?)", + schemaName, tableName); + return count == 1; + } + + @Override + public void dropTableIfExists(JdbcDatabase database, String schemaName, String tableName) throws Exception { + if (tableExists(database, schemaName, tableName)) { + try { + final String query = String.format("DROP TABLE %s.%s", schemaName, tableName); + database.execute(query); + } catch (Exception e) { + LOGGER.error(String.format("Error dropping table %s.%s", schemaName, tableName), e); + throw e; + } + } + } + + @Override + public String truncateTableQuery(String schemaName, String tableName) { + return String.format("DELETE FROM %s.%s\n", schemaName, tableName); + } + + @Override + public void insertRecords(JdbcDatabase database, List records, String schemaName, String tempTableName) + throws Exception { + final String tableName = String.format("%s.%s", schemaName, tempTableName); + final String columns = String.format("(%s, %s, %s)", + OracleDestination.COLUMN_NAME_AB_ID, OracleDestination.COLUMN_NAME_DATA, OracleDestination.COLUMN_NAME_EMITTED_AT); + final String recordQueryComponent = "(?, ?, ?)\n"; + insertRawRecordsInSingleQuery(tableName, columns, recordQueryComponent, database, records, UUID::randomUUID); + } + + // Adapted from SqlUtils.insertRawRecordsInSingleQuery to meet some needs specific to Oracle syntax + private static void insertRawRecordsInSingleQuery(String tableName, + String columns, + String recordQueryComponent, + JdbcDatabase jdbcDatabase, + List records, + Supplier uuidSupplier) + throws SQLException { + if (records.isEmpty()) { + return; + } + + jdbcDatabase.execute(connection -> { + + // Strategy: We want to use PreparedStatement because it handles binding values to the SQL query + // (e.g. handling formatting timestamps). A PreparedStatement statement is created by supplying the + // full SQL string at creation time. Then subsequently specifying which values are bound to the + // string. Thus there will be two loops below. + // 1) Loop over records to build the full string. + // 2) Loop over the records and bind the appropriate values to the string. + // + // The "SELECT 1 FROM DUAL" at the end is a formality to satisfy the needs of the Oracle syntax. + // (see https://stackoverflow.com/a/93724 for details) + final StringBuilder sql = new StringBuilder("INSERT ALL "); + records.forEach(r -> sql.append(String.format("INTO %s %s VALUES %s", tableName, columns, recordQueryComponent))); + sql.append(" SELECT 1 FROM DUAL"); + final String query = sql.toString(); + + try (final PreparedStatement statement = connection.prepareStatement(query)) { + // second loop: bind values to the SQL string. + int i = 1; + for (final AirbyteRecordMessage message : records) { + // 1-indexed + statement.setString(i, uuidSupplier.get().toString()); + statement.setString(i + 1, Jsons.serialize(message.getData())); + statement.setTimestamp(i + 2, Timestamp.from(Instant.ofEpochMilli(message.getEmittedAt()))); + i += 3; + } + + statement.execute(); + } + }); + } + + @Override + public String copyTableQuery(String schemaName, String sourceTableName, String destinationTableName) { + return String.format("INSERT INTO %s.%s SELECT * FROM %s.%s\n", schemaName, destinationTableName, schemaName, sourceTableName); + } + + @Override + public void executeTransaction(JdbcDatabase database, List queries) throws Exception { + String SQL = "BEGIN\n COMMIT;\n" + String.join(";\n", queries) + "; \nCOMMIT; \nEND;"; + database.execute(SQL); + } + + @Override + public boolean isValidData(String data) { + return true; + } + + @Override + public boolean isSchemaRequired() { + return true; + } + +} diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json new file mode 100644 index 000000000000..6fe6a6b30be8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/src/main/resources/spec.json @@ -0,0 +1,57 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/oracle", + "supportsIncremental": true, + "supported_destination_sync_modes": ["overwrite", "append"], + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Oracle Destination Spec", + "type": "object", + "required": ["host", "port", "username", "database"], + "additionalProperties": false, + "properties": { + "host": { + "title": "Host", + "description": "Hostname of the database.", + "type": "string", + "order": 0 + }, + "port": { + "title": "Port", + "description": "Port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 1521, + "examples": ["1521"], + "order": 1 + }, + "database": { + "title": "DB Name", + "description": "Name of the database.", + "type": "string", + "order": 2 + }, + "username": { + "title": "User", + "description": "Username to use to access the database. This user must have CREATE USER privileges in the database.", + "type": "string", + "order": 3 + }, + "password": { + "title": "Password", + "description": "Password associated with the username.", + "type": "string", + "airbyte_secret": true, + "order": 4 + }, + "schema": { + "title": "Default Schema", + "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"airbyte\". In Oracle, schemas and users are the same thing, so the \"user\" parameter is used as the login credentials and this is used for the default Airbyte message schema.", + "type": "string", + "examples": ["airbyte"], + "default": "airbyte", + "order": 5 + } + } + } +} diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleIntegrationTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleIntegrationTest.java new file mode 100644 index 000000000000..ffca5c39c34d --- /dev/null +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleIntegrationTest.java @@ -0,0 +1,210 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.oracle; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.db.Database; +import io.airbyte.db.Databases; +import io.airbyte.integrations.destination.ExtendedNameTransformer; +import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import org.jooq.JSONFormat; +import org.jooq.JSONFormat.RecordFormat; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.OracleContainer; + +public class OracleIntegrationTest extends DestinationAcceptanceTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(OracleIntegrationTest.class); + private static final JSONFormat JSON_FORMAT = new JSONFormat().recordFormat(RecordFormat.OBJECT); + + private static OracleContainer db; + private ExtendedNameTransformer namingResolver = new OracleNameTransformer(); + private JsonNode config; + + @BeforeAll + protected static void init() { + db = new OracleContainer("epiclabs/docker-oracle-xe-11g"); + db.start(); + } + + @Override + protected String getImageName() { + return "airbyte/destination-oracle:dev"; + } + + private JsonNode getConfig(OracleContainer db) { + return Jsons.jsonNode(ImmutableMap.builder() + .put("host", db.getHost()) + .put("port", db.getFirstMappedPort()) + .put("username", db.getUsername()) + .put("password", db.getPassword()) + .put("schema", "testSchema") + .put("sid", db.getSid()) + .build()); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected JsonNode getFailCheckConfig() { + return Jsons.jsonNode(ImmutableMap.builder() + .put("host", db.getHost()) + .put("username", db.getUsername()) + .put("password", "wrong password") + .put("schema", "public") + .put("port", db.getFirstMappedPort()) + .put("sid", db.getSid()) + .build()); + } + + @Override + protected List retrieveRecords(TestDestinationEnv env, String streamName, String namespace, JsonNode streamSchema) throws Exception { + return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + .stream() + .map(r -> Jsons.deserialize(r.get(OracleDestination.COLUMN_NAME_DATA).asText())) + .collect(Collectors.toList()); + } + + @Override + protected boolean implementsBasicNormalization() { + return false; + } + + @Override + protected boolean implementsNamespaces() { + return true; + } + + @Override + protected List retrieveNormalizedRecords(TestDestinationEnv env, String streamName, String namespace) + throws Exception { + String tableName = namingResolver.getIdentifier(streamName); + return retrieveRecordsFromTable(tableName, namespace); + } + + @Override + protected List resolveIdentifier(String identifier) { + final List result = new ArrayList<>(); + final String resolved = namingResolver.getIdentifier(identifier); + result.add(identifier); + result.add(resolved); + if (!resolved.startsWith("\"")) { + result.add(resolved.toLowerCase()); + result.add(resolved.toUpperCase()); + } + return result; + } + + private List retrieveRecordsFromTable(String tableName, String schemaName) throws SQLException { + List result = Databases.createOracleDatabase(db.getUsername(), db.getPassword(), db.getJdbcUrl()) + .query(ctx -> ctx + .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC", schemaName, tableName, OracleDestination.COLUMN_NAME_EMITTED_AT)) + .stream() + .collect(Collectors.toList())); + return result + .stream() + .map(r -> r.formatJSON(JSON_FORMAT)) + .map(Jsons::deserialize) + .collect(Collectors.toList()); + } + + private static Database getDatabase(JsonNode config) { + // todo (cgardens) - rework this abstraction so that we do not have to pass a null into the + // constructor. at least explicitly handle it, even if the impl doesn't change. + return Databases.createDatabase( + config.get("username").asText(), + config.get("password").asText(), + String.format("jdbc:oracle:thin:@//%s:%s/%s", + config.get("host").asText(), + config.get("port").asText(), + config.get("sid").asText()), + "oracle.jdbc.driver.OracleDriver", + null); + } + + private List allTables; + + private List getAllTables(Database db) { + try { + return db.query(ctx -> ctx.fetch("select OWNER, TABLE_NAME from ALL_TABLES where upper(TABLESPACE_NAME) = 'USERS'") + .stream() + .map(r -> String.format("%s.%s", r.get("OWNER"), r.get("TABLE_NAME"))) + .collect(Collectors.toList())); + } catch (SQLException e) { + LOGGER.error("Error while cleaning up test.", e); + return null; + } + } + + @Override + protected void setup(TestDestinationEnv testEnv) throws SQLException { + config = getConfig(db); + + final Database database = getDatabase(config); + database.query(ctx -> { + ctx.execute("alter database default tablespace users"); + return null; + }); + allTables = getAllTables(database); + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) { + config = getConfig(db); + + final Database database = getDatabase(config); + var tables = getAllTables(database); + tables.removeAll(allTables); + try { + for (String table : tables) { + database.query(ctx -> { + ctx.execute("drop table " + table); + return null; + }); + } + } catch (SQLException e) { + LOGGER.error("Error while cleaning up test.", e); + } + } + + @AfterAll + static void cleanUp() { + db.stop(); + db.close(); + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/.dockerignore b/airbyte-integrations/connectors/destination-s3/.dockerignore new file mode 100644 index 000000000000..65c7d0ad3e73 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile new file mode 100644 index 000000000000..7af717908c50 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/Dockerfile @@ -0,0 +1,11 @@ +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte +ENV APPLICATION destination-s3 + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle new file mode 100644 index 000000000000..e1bd8103e31d --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/build.gradle @@ -0,0 +1,26 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.destination.s3.S3Destination' +} + +dependencies { + implementation project(':airbyte-config:models') + implementation project(':airbyte-protocol:models') + implementation project(':airbyte-integrations:bases:base-java') + implementation project(':airbyte-integrations:connectors:destination-jdbc') + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + + implementation 'com.amazonaws:aws-java-sdk-s3:1.11.978' + implementation 'org.apache.commons:commons-csv:1.4' + implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' + + testImplementation 'org.apache.commons:commons-lang3:3.11' + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-s3') +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Consumer.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Consumer.java new file mode 100644 index 000000000000..84b8efb4a2b4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Consumer.java @@ -0,0 +1,125 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteMessage.Type; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import java.sql.Timestamp; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.function.Consumer; + +public class S3Consumer extends FailureTrackingAirbyteMessageConsumer { + + private final S3DestinationConfig s3DestinationConfig; + private final ConfiguredAirbyteCatalog configuredCatalog; + private final S3OutputFormatterFactory formatterFactory; + private final Consumer outputRecordCollector; + private final Map streamNameAndNamespaceToFormatters; + + private AirbyteMessage lastStateMessage = null; + + public S3Consumer(S3DestinationConfig s3DestinationConfig, + ConfiguredAirbyteCatalog configuredCatalog, + S3OutputFormatterFactory formatterFactory, + Consumer outputRecordCollector) { + this.s3DestinationConfig = s3DestinationConfig; + this.configuredCatalog = configuredCatalog; + this.formatterFactory = formatterFactory; + this.outputRecordCollector = outputRecordCollector; + this.streamNameAndNamespaceToFormatters = new HashMap<>(configuredCatalog.getStreams().size()); + } + + @Override + protected void startTracked() throws Exception { + AWSCredentials awsCreds = new BasicAWSCredentials(s3DestinationConfig.getAccessKeyId(), + s3DestinationConfig.getSecretAccessKey()); + AmazonS3 s3Client = AmazonS3ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) + .withRegion(s3DestinationConfig.getBucketRegion()) + .build(); + Timestamp uploadTimestamp = new Timestamp(System.currentTimeMillis()); + + for (ConfiguredAirbyteStream configuredStream : configuredCatalog.getStreams()) { + S3OutputFormatter formatter = formatterFactory + .create(s3DestinationConfig, s3Client, configuredStream, uploadTimestamp); + formatter.initialize(); + + AirbyteStream stream = configuredStream.getStream(); + AirbyteStreamNameNamespacePair streamNamePair = AirbyteStreamNameNamespacePair + .fromAirbyteSteam(stream); + streamNameAndNamespaceToFormatters.put(streamNamePair, formatter); + } + } + + @Override + protected void acceptTracked(AirbyteMessage airbyteMessage) throws Exception { + if (airbyteMessage.getType() == Type.STATE) { + this.lastStateMessage = airbyteMessage; + return; + } else if (airbyteMessage.getType() != Type.RECORD) { + return; + } + + AirbyteRecordMessage recordMessage = airbyteMessage.getRecord(); + AirbyteStreamNameNamespacePair pair = AirbyteStreamNameNamespacePair + .fromRecordMessage(recordMessage); + + if (!streamNameAndNamespaceToFormatters.containsKey(pair)) { + throw new IllegalArgumentException( + String.format( + "Message contained record from a stream that was not in the catalog. \ncatalog: %s , \nmessage: %s", + Jsons.serialize(configuredCatalog), Jsons.serialize(recordMessage))); + } + + UUID id = UUID.randomUUID(); + streamNameAndNamespaceToFormatters.get(pair).write(id, recordMessage); + } + + @Override + protected void close(boolean hasFailed) throws Exception { + for (S3OutputFormatter handler : streamNameAndNamespaceToFormatters.values()) { + handler.close(hasFailed); + } + // S3 stream uploader is all or nothing if a failure happens in the destination. + if (!hasFailed) { + outputRecordCollector.accept(lastStateMessage); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java new file mode 100644 index 000000000000..5bc4e711d403 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java @@ -0,0 +1,72 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.BaseConnector; +import io.airbyte.integrations.base.AirbyteMessageConsumer; +import io.airbyte.integrations.base.Destination; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.destination.jdbc.copy.s3.S3Config; +import io.airbyte.integrations.destination.jdbc.copy.s3.S3StreamCopier; +import io.airbyte.protocol.models.AirbyteConnectionStatus; +import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class S3Destination extends BaseConnector implements Destination { + + private static final Logger LOGGER = LoggerFactory.getLogger(S3Destination.class); + + public static void main(String[] args) throws Exception { + new IntegrationRunner(new S3Destination()).run(args); + } + + @Override + public AirbyteConnectionStatus check(JsonNode config) { + try { + S3StreamCopier.attemptS3WriteAndDelete(S3Config.getS3Config(config)); + return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); + } catch (Exception e) { + LOGGER.error("Exception attempting to access the S3 bucket: ", e); + return new AirbyteConnectionStatus() + .withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage("Could not connect to the S3 bucket with the provided configuration. \n" + e + .getMessage()); + } + } + + @Override + public AirbyteMessageConsumer getConsumer(JsonNode config, + ConfiguredAirbyteCatalog configuredCatalog, + Consumer outputRecordCollector) { + S3OutputFormatterFactory formatterFactory = new S3OutputFormatterProductionFactory(); + return new S3Consumer(S3DestinationConfig.getS3DestinationConfig(config), configuredCatalog, formatterFactory, outputRecordCollector); + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java new file mode 100644 index 000000000000..64c9899905fb --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java @@ -0,0 +1,87 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import com.fasterxml.jackson.databind.JsonNode; + +public class S3DestinationConfig { + + private final String bucketName; + private final String bucketPath; + private final String bucketRegion; + private final String accessKeyId; + private final String secretAccessKey; + private final S3FormatConfig formatConfig; + + public S3DestinationConfig( + String bucketName, + String bucketPath, + String bucketRegion, + String accessKeyId, + String secretAccessKey, + S3FormatConfig formatConfig) { + this.bucketName = bucketName; + this.bucketPath = bucketPath; + this.bucketRegion = bucketRegion; + this.accessKeyId = accessKeyId; + this.secretAccessKey = secretAccessKey; + this.formatConfig = formatConfig; + } + + public static S3DestinationConfig getS3DestinationConfig(JsonNode config) { + return new S3DestinationConfig( + config.get("s3_bucket_name").asText(), + config.get("s3_bucket_path").asText(), + config.get("s3_bucket_region").asText(), + config.get("access_key_id").asText(), + config.get("secret_access_key").asText(), + S3FormatConfigs.getS3FormatConfig(config)); + } + + public String getBucketName() { + return bucketName; + } + + public String getBucketPath() { + return bucketPath; + } + + public String getBucketRegion() { + return bucketRegion; + } + + public String getAccessKeyId() { + return accessKeyId; + } + + public String getSecretAccessKey() { + return secretAccessKey; + } + + public S3FormatConfig getFormatConfig() { + return formatConfig; + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java new file mode 100644 index 000000000000..bce005bda796 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java @@ -0,0 +1,45 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import java.text.DateFormat; +import java.text.SimpleDateFormat; + +public final class S3DestinationConstants { + + // These parameters are used by {@link StreamTransferManager}. + // See this doc about how they affect memory usage: + // https://alexmojaki.github.io/s3-stream-upload/javadoc/apidocs/alex/mojaki/s3upload/StreamTransferManager.html + // Total memory = (numUploadThreads + queueCapacity) * partSize + numStreams * (partSize + 6MB) + // = 31 MB at current configurations + public static final int DEFAULT_UPLOAD_THREADS = 2; + public static final int DEFAULT_QUEUE_CAPACITY = 2; + public static final int DEFAULT_PART_SIZE_MD = 5; + public static final int DEFAULT_NUM_STREAMS = 1; + public static final DateFormat YYYY_MM_DD_FORMAT = new SimpleDateFormat("yyyy_MM_dd"); + + private S3DestinationConstants() {} + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Format.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Format.java new file mode 100644 index 000000000000..10cb592cb88d --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Format.java @@ -0,0 +1,29 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +public enum S3Format { + CSV +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java new file mode 100644 index 000000000000..9b7bcbe8609a --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java @@ -0,0 +1,31 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +public interface S3FormatConfig { + + S3Format getFormat(); + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java new file mode 100644 index 000000000000..693d628f44d1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java @@ -0,0 +1,46 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; +import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig.Flattening; + +public class S3FormatConfigs { + + public static S3FormatConfig getS3FormatConfig(JsonNode config) { + JsonNode formatConfig = config.get("format"); + S3Format formatType = S3Format.valueOf(formatConfig.get("format_type").asText()); + + if (formatType == S3Format.CSV) { + Flattening flattening = Flattening.fromValue(formatConfig.get("flattening").asText()); + return new S3CsvFormatConfig(flattening); + } + + throw new RuntimeException("Unexpected output format: " + Jsons.serialize(config)); + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatter.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatter.java new file mode 100644 index 000000000000..4be7b6f77b4e --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatter.java @@ -0,0 +1,52 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.io.IOException; +import java.util.UUID; + +/** + * {@link S3OutputFormatter} is responsible for writing Airbyte stream data to an S3 location in a + * specific format. + */ +public interface S3OutputFormatter { + + /** + * Prepare an S3 writer for the stream. + */ + void initialize() throws IOException; + + /** + * Write an Airbyte record message to an S3 object. + */ + void write(UUID id, AirbyteRecordMessage recordMessage) throws IOException; + + /** + * Close the S3 writer for the stream. + */ + void close(boolean hasFailed) throws IOException; + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterFactory.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterFactory.java new file mode 100644 index 000000000000..f08dce169a24 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterFactory.java @@ -0,0 +1,43 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import com.amazonaws.services.s3.AmazonS3; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import java.io.IOException; +import java.sql.Timestamp; + +/** + * Create different {@link S3OutputFormatter} based on {@link S3DestinationConfig}. + */ +public interface S3OutputFormatterFactory { + + S3OutputFormatter create(S3DestinationConfig config, + AmazonS3 s3Client, + ConfiguredAirbyteStream configuredStream, + Timestamp uploadTimestamp) + throws IOException; + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterProductionFactory.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterProductionFactory.java new file mode 100644 index 000000000000..582f3cfd8ea9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterProductionFactory.java @@ -0,0 +1,49 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import com.amazonaws.services.s3.AmazonS3; +import io.airbyte.integrations.destination.s3.csv.S3CsvOutputFormatter; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import java.io.IOException; +import java.sql.Timestamp; + +public class S3OutputFormatterProductionFactory implements S3OutputFormatterFactory { + + @Override + public S3OutputFormatter create(S3DestinationConfig config, + AmazonS3 s3Client, + ConfiguredAirbyteStream configuredStream, + Timestamp uploadTimestamp) + throws IOException { + S3Format format = config.getFormatConfig().getFormat(); + if (format == S3Format.CSV) { + return new S3CsvOutputFormatter(config, s3Client, configuredStream, uploadTimestamp); + } + + throw new RuntimeException("Unexpected S3 destination format: " + format); + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java new file mode 100644 index 000000000000..b3882322afd8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java @@ -0,0 +1,49 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; + +/** + * CSV data row = ID column + timestamp column + record columns. This class takes care of the first + * two columns, which is shared by downstream implementations. + */ +public abstract class BaseSheetGenerator implements CsvSheetGenerator { + + public List getDataRow(UUID id, AirbyteRecordMessage recordMessage) { + List data = new LinkedList<>(); + data.add(id); + data.add(recordMessage.getEmittedAt()); + data.addAll(getRecordColumns(recordMessage.getData())); + return data; + } + + abstract List getRecordColumns(JsonNode json); + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java new file mode 100644 index 000000000000..7f6bea756b65 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java @@ -0,0 +1,58 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig.Flattening; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import java.util.List; +import java.util.UUID; + +/** + * This class takes case of the generation of the CSV data sheet, including the header row and the + * data row. + */ +public interface CsvSheetGenerator { + + List getHeaderRow(); + + List getDataRow(UUID id, AirbyteRecordMessage recordMessage); + + final class Factory { + + public static CsvSheetGenerator create(JsonNode jsonSchema, S3CsvFormatConfig formatConfig) { + if (formatConfig.getFlattening() == Flattening.NO) { + return new NoFlatteningSheetGenerator(); + } else if (formatConfig.getFlattening() == Flattening.ROOT_LEVEL) { + return new RootLevelFlatteningSheetGenerator(jsonSchema); + } else { + throw new IllegalArgumentException( + "Unexpected flattening config: " + formatConfig.getFlattening()); + } + } + + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerators.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerators.java new file mode 100644 index 000000000000..7d32b2e860d7 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerators.java @@ -0,0 +1,29 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +public class CsvSheetGenerators { + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java new file mode 100644 index 000000000000..402a45abf808 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java @@ -0,0 +1,52 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.JavaBaseConstants; +import java.util.Collections; +import java.util.List; + +public class NoFlatteningSheetGenerator extends BaseSheetGenerator implements CsvSheetGenerator { + + @Override + public List getHeaderRow() { + return Lists.newArrayList( + JavaBaseConstants.COLUMN_NAME_AB_ID, + JavaBaseConstants.COLUMN_NAME_EMITTED_AT, + JavaBaseConstants.COLUMN_NAME_DATA); + } + + /** + * When no flattening is needed, the record column is just one json blob. + */ + @Override + List getRecordColumns(JsonNode json) { + return Collections.singletonList(Jsons.serialize(json)); + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java new file mode 100644 index 000000000000..01d570b09e01 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java @@ -0,0 +1,81 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.MoreIterators; +import io.airbyte.integrations.base.JavaBaseConstants; +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +public class RootLevelFlatteningSheetGenerator extends BaseSheetGenerator implements CsvSheetGenerator { + + /** + * Keep a header list to iterate the input json object with a defined order. + */ + private final List recordHeaders; + + public RootLevelFlatteningSheetGenerator(JsonNode jsonSchema) { + this.recordHeaders = MoreIterators.toList(jsonSchema.get("properties").fieldNames()) + .stream().sorted().collect(Collectors.toList());; + } + + @Override + public List getHeaderRow() { + List headers = Lists.newArrayList(JavaBaseConstants.COLUMN_NAME_AB_ID, + JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + headers.addAll(recordHeaders); + return headers; + } + + /** + * With root level flattening, the record columns are the first level fields of the json. + */ + @Override + List getRecordColumns(JsonNode json) { + List values = new LinkedList<>(); + for (String field : recordHeaders) { + JsonNode value = json.get(field); + if (value == null) { + values.add(""); + } else if (value.isValueNode()) { + // Call asText method on value nodes so that proper string + // representation of json values can be returned by Jackson. + // Otherwise, CSV printer will just call the toString method, + // which can be problematic (e.g. text node will have extra + // double quotation marks around its text value). + values.add(value.asText()); + } else { + values.add(Jsons.serialize(value)); + } + } + + return values; + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java new file mode 100644 index 000000000000..bdc35f8cd53f --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java @@ -0,0 +1,77 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +import com.fasterxml.jackson.annotation.JsonCreator; +import io.airbyte.integrations.destination.s3.S3Format; +import io.airbyte.integrations.destination.s3.S3FormatConfig; +import java.util.Locale; + +public class S3CsvFormatConfig implements S3FormatConfig { + + public enum Flattening { + + // These values must match the format / csv_flattening enum values in spec.json. + NO("No flattening"), + ROOT_LEVEL("Root level flattening"); + + private final String value; + + Flattening(String value) { + this.value = value; + } + + @JsonCreator + public static Flattening fromValue(String value) { + for (Flattening f : Flattening.values()) { + if (f.value.toLowerCase(Locale.ROOT).equals(value.toLowerCase())) { + return f; + } + } + throw new IllegalArgumentException("Unexpected value: " + value); + } + + public String getValue() { + return value; + } + + } + + private final Flattening flattening; + + public S3CsvFormatConfig(Flattening flattening) { + this.flattening = flattening; + } + + @Override + public S3Format getFormat() { + return S3Format.CSV; + } + + public Flattening getFlattening() { + return flattening; + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatter.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatter.java new file mode 100644 index 000000000000..d7f7fb31be91 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatter.java @@ -0,0 +1,192 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +import static io.airbyte.integrations.destination.s3.S3DestinationConstants.YYYY_MM_DD_FORMAT; + +import alex.mojaki.s3upload.MultiPartOutputStream; +import alex.mojaki.s3upload.StreamTransferManager; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; +import com.amazonaws.services.s3.model.DeleteObjectsResult; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.integrations.destination.ExtendedNameTransformer; +import io.airbyte.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.integrations.destination.s3.S3DestinationConstants; +import io.airbyte.integrations.destination.s3.S3OutputFormatter; +import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.DestinationSyncMode; +import java.io.IOException; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.sql.Timestamp; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; +import org.apache.commons.csv.QuoteMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class S3CsvOutputFormatter implements S3OutputFormatter { + + private static final Logger LOGGER = LoggerFactory.getLogger(S3CsvOutputFormatter.class); + private static final ExtendedNameTransformer NAME_TRANSFORMER = new ExtendedNameTransformer(); + + private final S3DestinationConfig config; + private final S3CsvFormatConfig formatConfig; + private final AmazonS3 s3Client; + private final AirbyteStream stream; + private final DestinationSyncMode syncMode; + private final CsvSheetGenerator csvSheetGenerator; + private final String outputPrefix; + private final StreamTransferManager uploadManager; + private final MultiPartOutputStream outputStream; + private final CSVPrinter csvPrinter; + + public S3CsvOutputFormatter(S3DestinationConfig config, + AmazonS3 s3Client, + ConfiguredAirbyteStream configuredStream, + Timestamp uploadTimestamp) + throws IOException { + this.config = config; + this.formatConfig = (S3CsvFormatConfig) config.getFormatConfig(); + this.s3Client = s3Client; + this.stream = configuredStream.getStream(); + this.syncMode = configuredStream.getDestinationSyncMode(); + this.csvSheetGenerator = CsvSheetGenerator.Factory.create(configuredStream.getStream().getJsonSchema(), formatConfig); + + // prefix: // + // filename: -.csv + // full path: // + this.outputPrefix = getOutputPrefix(config.getBucketPath(), stream); + String outputFilename = getOutputFilename(uploadTimestamp); + String objectKey = String.join("/", outputPrefix, outputFilename); + + LOGGER.info("Full S3 path for stream '{}': {}/{}", stream.getName(), config.getBucketName(), + objectKey); + + // The stream transfer manager lets us greedily stream into S3. The native AWS SDK does not + // have support for streaming multipart uploads. The alternative is first writing the entire + // output to disk before loading into S3. This is not feasible with large input. + // Data is chunked into parts during the upload. A part is sent off to a queue to be uploaded + // once it has reached it's configured part size. + // See {@link S3DestinationConstants} for memory usage calculation. + this.uploadManager = new StreamTransferManager(config.getBucketName(), objectKey, s3Client) + .numStreams(S3DestinationConstants.DEFAULT_NUM_STREAMS) + .queueCapacity(S3DestinationConstants.DEFAULT_QUEUE_CAPACITY) + .numUploadThreads(S3DestinationConstants.DEFAULT_UPLOAD_THREADS) + .partSize(S3DestinationConstants.DEFAULT_PART_SIZE_MD); + // We only need one output stream as we only have one input stream. This is reasonably performant. + this.outputStream = uploadManager.getMultiPartOutputStreams().get(0); + this.csvPrinter = new CSVPrinter(new PrintWriter(outputStream, true, StandardCharsets.UTF_8), + CSVFormat.DEFAULT.withQuoteMode(QuoteMode.ALL) + .withHeader(csvSheetGenerator.getHeaderRow().toArray(new String[0]))); + } + + static String getOutputPrefix(String bucketPath, AirbyteStream stream) { + return getOutputPrefix(bucketPath, stream.getNamespace(), stream.getName()); + } + + @VisibleForTesting + public static String getOutputPrefix(String bucketPath, String namespace, String streamName) { + List paths = new LinkedList<>(); + + if (bucketPath != null) { + paths.add(bucketPath); + } + if (namespace != null) { + paths.add(NAME_TRANSFORMER.convertStreamName(namespace)); + } + paths.add(NAME_TRANSFORMER.convertStreamName(streamName)); + + return String.join("/", paths); + } + + static String getOutputFilename(Timestamp timestamp) { + return String.format("%s_%d_0.csv", YYYY_MM_DD_FORMAT.format(timestamp), timestamp.getTime()); + } + + /** + *
  • 1. Create bucket if necessary.
  • + *
  • 2. Under OVERWRITE mode, delete all objects with the output prefix.
  • + */ + @Override + public void initialize() { + String bucket = config.getBucketName(); + if (!s3Client.doesBucketExistV2(bucket)) { + LOGGER.info("Bucket {} does not exist; creating...", bucket); + s3Client.createBucket(bucket); + LOGGER.info("Bucket {} has been created.", bucket); + } + + if (syncMode == DestinationSyncMode.OVERWRITE) { + LOGGER.info("Overwrite mode"); + List keysToDelete = new LinkedList<>(); + List objects = s3Client.listObjects(bucket, outputPrefix) + .getObjectSummaries(); + for (S3ObjectSummary object : objects) { + keysToDelete.add(new KeyVersion(object.getKey())); + } + + if (keysToDelete.size() > 0) { + LOGGER.info("Purging non-empty output path for stream '{}' under OVERWRITE mode...", + stream.getName()); + DeleteObjectsResult result = s3Client + .deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keysToDelete)); + LOGGER.info("Deleted {} file(s) for stream '{}'.", result.getDeletedObjects().size(), + stream.getName()); + } + } + } + + @Override + public void write(UUID id, AirbyteRecordMessage recordMessage) throws IOException { + csvPrinter.printRecord(csvSheetGenerator.getDataRow(id, recordMessage)); + } + + @Override + public void close(boolean hasFailed) throws IOException { + if (hasFailed) { + LOGGER.warn("Failure detected. Aborting upload of stream '{}'...", stream.getName()); + csvPrinter.close(); + outputStream.close(); + uploadManager.abort(); + LOGGER.warn("Upload of stream '{}' aborted.", stream.getName()); + } else { + LOGGER.info("Uploading remaining data for stream '{}'.", stream.getName()); + csvPrinter.close(); + outputStream.close(); + uploadManager.complete(); + LOGGER.info("Upload completed for stream '{}'.", stream.getName()); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json new file mode 100644 index 000000000000..4d9808c04078 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json @@ -0,0 +1,103 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/s3", + "supportsIncremental": true, + "supported_destination_sync_modes": ["overwrite", "append"], + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "S3 Destination Spec", + "type": "object", + "required": [ + "s3_bucket_name", + "s3_bucket_path", + "s3_bucket_region", + "access_key_id", + "secret_access_key", + "format" + ], + "additionalProperties": false, + "properties": { + "s3_bucket_name": { + "title": "S3 Bucket Name", + "type": "string", + "description": "The name of the S3 bucket.", + "examples": ["airbyte_sync"] + }, + "s3_bucket_path": { + "description": "Directory under the S3 bucket where data will be written.", + "type": "string", + "examples": ["data_sync/test"] + }, + "s3_bucket_region": { + "title": "S3 Bucket Region", + "type": "string", + "default": "", + "description": "The region of the S3 bucket.", + "enum": [ + "", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "af-south-1", + "ap-east-1", + "ap-south-1", + "ap-northeast-1", + "ap-northeast-2", + "ap-northeast-3", + "ap-southeast-1", + "ap-southeast-2", + "ca-central-1", + "cn-north-1", + "cn-northwest-1", + "eu-central-1", + "eu-north-1", + "eu-south-1", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "sa-east-1", + "me-south-1" + ] + }, + "access_key_id": { + "type": "string", + "description": "The access key id to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket.", + "title": "S3 Key Id", + "airbyte_secret": true, + "examples": ["A012345678910EXAMPLE"] + }, + "secret_access_key": { + "type": "string", + "description": "The corresponding secret to the access key id.", + "title": "S3 Access Key", + "airbyte_secret": true, + "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"] + }, + "format": { + "title": "Output Format", + "type": "object", + "description": "Output data format", + "oneOf": [ + { + "title": "CSV: Comma-separated values", + "required": ["format_type", "flattening"], + "properties": { + "format_type": { + "type": "string", + "enum": ["CSV"], + "default": "CSV" + }, + "flattening": { + "type": "string", + "title": "Flattening", + "description": "Whether the input json data should be flattened in the output CSV. Please refer to docs for details.", + "default": "No flattening", + "enum": ["No flattening", "Root level flattening"] + } + } + } + ] + } + } + } +} diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java new file mode 100644 index 000000000000..30df1d5d184c --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java @@ -0,0 +1,243 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; +import com.amazonaws.services.s3.model.DeleteObjectsResult; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.destination.s3.csv.S3CsvOutputFormatter; +import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.csv.QuoteMode; +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class S3CsvDestinationAcceptanceTest extends DestinationAcceptanceTest { + + private static final Logger LOGGER = LoggerFactory + .getLogger(S3CsvDestinationAcceptanceTest.class); + private static final ObjectMapper mapper = new ObjectMapper(); + + private JsonNode configJson; + private S3DestinationConfig config; + private AmazonS3 s3Client; + + /** + * Convert json_schema to a map from field name to field types. + */ + private static Map getFieldTypes(JsonNode streamSchema) { + Map fieldTypes = new HashMap<>(); + JsonNode fieldDefinitions = streamSchema.get("properties"); + Iterator> iterator = fieldDefinitions.fields(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + fieldTypes.put(entry.getKey(), entry.getValue().get("type").asText()); + } + return fieldTypes; + } + + private static Optional parseNumber(String input) { + try { + double number = Double.parseDouble(input); + if (number % 1 == 0) { + return Optional.of((int) number); + } else { + return Optional.of(number); + } + } catch (Exception e) { + return Optional.empty(); + } + } + + private static JsonNode getJsonNode(Map input, Map fieldTypes) { + ObjectNode json = mapper.createObjectNode(); + + if (input.containsKey(JavaBaseConstants.COLUMN_NAME_DATA)) { + return Jsons.deserialize(input.get(JavaBaseConstants.COLUMN_NAME_DATA)); + } + + for (Map.Entry entry : input.entrySet()) { + String key = entry.getKey(); + if (key.equals(JavaBaseConstants.COLUMN_NAME_AB_ID) || key + .equals(JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) { + continue; + } + String value = entry.getValue(); + if (value == null || value.equals("")) { + continue; + } + String type = fieldTypes.get(key); + switch (type) { + case "boolean" -> json.put(key, Boolean.valueOf(value)); + case "integer" -> json.put(key, Integer.valueOf(value)); + case "number" -> { + Optional numValue = parseNumber(value); + if (numValue.isPresent()) { + if (numValue.get() instanceof Integer) { + json.put(key, (int) numValue.get()); + } else { + json.put(key, (double) numValue.get()); + } + } else { + json.put(key, value); + } + } + default -> json.put(key, value); + } + } + return json; + } + + private static JsonNode getBaseConfigJson() { + return Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); + } + + @Override + protected String getImageName() { + return "airbyte/destination-s3:dev"; + } + + @Override + protected JsonNode getConfig() { + return configJson; + } + + @Override + protected JsonNode getFailCheckConfig() { + JsonNode baseJson = getBaseConfigJson(); + JsonNode failCheckJson = Jsons.clone(baseJson); + // invalid credential + ((ObjectNode) failCheckJson).put("access_key_id", "fake-key"); + ((ObjectNode) failCheckJson).put("secret_access_key", "fake-secret"); + return failCheckJson; + } + + @Override + protected List retrieveRecords(TestDestinationEnv testEnv, + String streamName, + String namespace, + JsonNode streamSchema) + throws IOException { + String outputPrefix = S3CsvOutputFormatter + .getOutputPrefix(config.getBucketPath(), namespace, streamName); + List objectSummaries = s3Client + .listObjects(config.getBucketName(), outputPrefix) + .getObjectSummaries() + .stream() + .sorted(Comparator.comparingLong(o -> o.getLastModified().getTime())) + .collect(Collectors.toList()); + LOGGER.info( + "All objects: {}", + objectSummaries.stream().map(o -> String.format("%s/%s", o.getBucketName(), o.getKey())).collect(Collectors.toList())); + + Map fieldTypes = getFieldTypes(streamSchema); + List jsonRecords = new LinkedList<>(); + + for (S3ObjectSummary objectSummary : objectSummaries) { + S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); + Reader in = new InputStreamReader(object.getObjectContent(), StandardCharsets.UTF_8); + Iterable records = CSVFormat.DEFAULT + .withQuoteMode(QuoteMode.NON_NUMERIC) + .withFirstRecordAsHeader() + .parse(in); + StreamSupport.stream(records.spliterator(), false) + .forEach(r -> jsonRecords.add(getJsonNode(r.toMap(), fieldTypes))); + } + + return jsonRecords; + } + + @Override + protected void setup(TestDestinationEnv testEnv) { + JsonNode baseConfigJson = getBaseConfigJson(); + // Set a random s3 bucket path for each integration test + JsonNode configJson = Jsons.clone(baseConfigJson); + String testBucketPath = String.format( + "%s_%s", + configJson.get("s3_bucket_path").asText(), + RandomStringUtils.randomAlphanumeric(5)); + ((ObjectNode) configJson).put("s3_bucket_path", testBucketPath); + this.configJson = configJson; + this.config = S3DestinationConfig.getS3DestinationConfig(configJson); + LOGGER.info("Test full path: {}/{}", config.getBucketName(), config.getBucketPath()); + + AWSCredentials awsCreds = new BasicAWSCredentials(config.getAccessKeyId(), + config.getSecretAccessKey()); + this.s3Client = AmazonS3ClientBuilder.standard() + .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) + .withRegion(config.getBucketRegion()) + .build(); + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) { + List keysToDelete = new LinkedList<>(); + List objects = s3Client + .listObjects(config.getBucketName(), config.getBucketPath()) + .getObjectSummaries(); + for (S3ObjectSummary object : objects) { + keysToDelete.add(new KeyVersion(object.getKey())); + } + + if (keysToDelete.size() > 0) { + LOGGER.info("Tearing down test bucket path: {}/{}", config.getBucketName(), + config.getBucketPath()); + DeleteObjectsResult result = s3Client + .deleteObjects(new DeleteObjectsRequest(config.getBucketName()).withKeys(keysToDelete)); + LOGGER.info("Deleted {} file(s).", result.getDeletedObjects().size()); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigsTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigsTest.java new file mode 100644 index 000000000000..dda674082064 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigsTest.java @@ -0,0 +1,58 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; +import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig.Flattening; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +@DisplayName("S3FormatConfigs") +public class S3FormatConfigsTest { + + private static final ObjectMapper mapper = new ObjectMapper(); + + @Test + @DisplayName("When CSV format is specified, it returns CSV format config") + public void testGetCsvS3FormatConfig() { + ObjectNode stubFormatConfig = mapper.createObjectNode(); + stubFormatConfig.put("format_type", S3Format.CSV.toString()); + stubFormatConfig.put("flattening", Flattening.ROOT_LEVEL.getValue()); + + ObjectNode stubConfig = mapper.createObjectNode(); + stubConfig.set("format", stubFormatConfig); + S3FormatConfig formatConfig = S3FormatConfigs.getS3FormatConfig(stubConfig); + assertEquals(formatConfig.getFormat(), S3Format.CSV); + assertTrue(formatConfig instanceof S3CsvFormatConfig); + S3CsvFormatConfig csvFormatConfig = (S3CsvFormatConfig) formatConfig; + assertEquals(csvFormatConfig.getFlattening(), Flattening.ROOT_LEVEL); + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGeneratorTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGeneratorTest.java new file mode 100644 index 000000000000..93b484b07bbe --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGeneratorTest.java @@ -0,0 +1,64 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +import static org.junit.jupiter.api.Assertions.assertLinesMatch; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; +import io.airbyte.integrations.base.JavaBaseConstants; +import java.util.Collections; +import org.junit.jupiter.api.Test; + +class NoFlatteningSheetGeneratorTest { + + private final ObjectMapper mapper = new ObjectMapper(); + private final NoFlatteningSheetGenerator sheetGenerator = new NoFlatteningSheetGenerator(); + + @Test + public void testGetHeaderRow() { + assertLinesMatch( + Lists.newArrayList( + JavaBaseConstants.COLUMN_NAME_AB_ID, + JavaBaseConstants.COLUMN_NAME_EMITTED_AT, + JavaBaseConstants.COLUMN_NAME_DATA), + sheetGenerator.getHeaderRow()); + } + + @Test + public void testGetRecordColumns() { + ObjectNode json = mapper.createObjectNode(); + json.set("Field 4", mapper.createObjectNode().put("Field 41", 15)); + json.put("Field 1", "A"); + json.put("Field 3", 71); + json.put("Field 2", true); + + assertLinesMatch( + Collections.singletonList("{\"Field 4\":{\"Field 41\":15},\"Field 1\":\"A\",\"Field 3\":71,\"Field 2\":true}"), + sheetGenerator.getRecordColumns(json)); + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGeneratorTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGeneratorTest.java new file mode 100644 index 000000000000..45b6325bc23f --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGeneratorTest.java @@ -0,0 +1,87 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +import static org.junit.jupiter.api.Assertions.assertLinesMatch; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Lists; +import io.airbyte.integrations.base.JavaBaseConstants; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class RootLevelFlatteningSheetGeneratorTest { + + private final static ObjectMapper MAPPER = new ObjectMapper(); + private final static ObjectNode SCHEMA = MAPPER.createObjectNode(); + static { + List fields = Lists.newArrayList("C", "B", "A", "c", "b", "a"); + Collections.shuffle(fields); + + ObjectNode schemaProperties = MAPPER.createObjectNode(); + for (String field : fields) { + schemaProperties.set(field, MAPPER.createObjectNode()); + } + + SCHEMA.set("properties", schemaProperties); + } + + private RootLevelFlatteningSheetGenerator sheetGenerator; + + @BeforeEach + public void createGenerator() { + this.sheetGenerator = new RootLevelFlatteningSheetGenerator(SCHEMA); + } + + @Test + public void testGetHeaderRow() { + assertLinesMatch( + Lists.newArrayList( + JavaBaseConstants.COLUMN_NAME_AB_ID, + JavaBaseConstants.COLUMN_NAME_EMITTED_AT, + "A", "B", "C", "a", "b", "c"), + sheetGenerator.getHeaderRow()); + } + + @Test + public void testGetRecordColumns() { + ObjectNode json = MAPPER.createObjectNode(); + // Field c is missing + json.put("C", 3); + json.put("B", "value B"); + json.set("A", MAPPER.createObjectNode().put("Field 41", 15)); + json.put("b", "value b"); + json.put("a", 1); + + assertLinesMatch( + // A, B, C, a, b, c + Lists.newArrayList("{\"Field 41\":15}", "value B", "3", "1", "value b", ""), + sheetGenerator.getRecordColumns(json)); + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java new file mode 100644 index 000000000000..35dde8bb970c --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +import static org.junit.jupiter.api.Assertions.*; + +import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig.Flattening; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +@DisplayName("S3CsvFormatConfig") +public class S3CsvFormatConfigTest { + + @Test + @DisplayName("Flattening enums can be created from value string") + public void testFlatteningCreationFromString() { + assertEquals(Flattening.NO, Flattening.fromValue("no flattening")); + assertEquals(Flattening.ROOT_LEVEL, Flattening.fromValue("root level flattening")); + try { + Flattening.fromValue("invalid flattening value"); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatterTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatterTest.java new file mode 100644 index 000000000000..c2d6972c3233 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatterTest.java @@ -0,0 +1,59 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3.csv; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.airbyte.protocol.models.AirbyteStream; +import java.sql.Timestamp; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +@DisplayName("S3CsvOutputFormatter") +class S3CsvOutputFormatterTest { + + @Test + @DisplayName("getOutputPrefix") + public void testGetOutputPrefix() { + // No namespace + assertEquals("bucket_path/stream_name", S3CsvOutputFormatter + .getOutputPrefix("bucket_path", new AirbyteStream().withName("stream_name"))); + + // With namespace + assertEquals("bucket_path/namespace/stream_name", S3CsvOutputFormatter + .getOutputPrefix("bucket_path", + new AirbyteStream().withNamespace("namespace").withName("stream_name"))); + } + + @Test + @DisplayName("getOutputFilename") + public void testGetOutputFilename() { + Timestamp timestamp = new Timestamp(1471461319000L); + assertEquals( + "2016_08_17_1471461319000_0.csv", + S3CsvOutputFormatter.getOutputFilename(timestamp)); + } + +} diff --git a/airbyte-integrations/connectors/source-amplitude/.dockerignore b/airbyte-integrations/connectors/source-amplitude/.dockerignore new file mode 100644 index 000000000000..b353347aedd0 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_amplitude +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-amplitude/CHANGELOG.md b/airbyte-integrations/connectors/source-amplitude/CHANGELOG.md new file mode 100644 index 000000000000..45fcf432379b --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +## 0.1.0 +Source implementation. diff --git a/airbyte-integrations/connectors/source-amplitude/Dockerfile b/airbyte-integrations/connectors/source-amplitude/Dockerfile new file mode 100644 index 000000000000..2e685a16e539 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_amplitude ./source_amplitude +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-amplitude diff --git a/airbyte-integrations/connectors/source-amplitude/README.md b/airbyte-integrations/connectors/source-amplitude/README.md new file mode 100644 index 000000000000..8d34f0612e99 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/README.md @@ -0,0 +1,129 @@ +# Amplitude Source + +This is the repository for the Amplitude source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/amplitude). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-amplitude:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/amplitude) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amplitude/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source amplitude test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-amplitude:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-amplitude:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-amplitude:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amplitude:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amplitude:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-amplitude:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-amplitude:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-amplitude:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml new file mode 100644 index 000000000000..8ba91112063b --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml @@ -0,0 +1,25 @@ +connector_image: airbyte/source-amplitude:dev +tests: + spec: + - spec_path: "source_amplitude/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/streams_with_output_records_catalog.json" + validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_without_events.json" + # Unable to use 'state_path' because Amplitude returns an error when specifying a date in the future. + # state_path: "integration_tests/abnormal_state.json" + cursor_paths: + active_users: [ "date" ] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_without_events.json" diff --git a/airbyte-integrations/connectors/source-amplitude/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-amplitude/acceptance-test-docker.sh new file mode 100644 index 000000000000..1425ff74f151 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-amplitude/build.gradle b/airbyte-integrations/connectors/source-amplitude/build.gradle new file mode 100644 index 000000000000..9196d6197402 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_amplitude' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/__init__.py b/airbyte-integrations/connectors/source-amplitude/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py new file mode 100644 index 000000000000..eeb4a2d3e02e --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..786041dfa7b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog.json @@ -0,0 +1,317 @@ +{ + "streams": [ + { + "stream": { + "name": "cohorts", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "appId": { + "type": ["null", "integer"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "definition": { + "type": ["null", "object"] + }, + "description": { + "type": ["null", "string"] + }, + "finished": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "owners": { + "type": ["null", "object"] + }, + "published": { + "type": ["null", "boolean"] + }, + "size": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "lastMod": { + "type": ["null", "integer"] + }, + "lastComputed": { + "type": ["null", "integer"] + }, + "hidden": { + "type": ["null", "boolean"] + }, + "is_predictive": { + "type": ["null", "boolean"] + }, + "is_official_content": { + "type": ["null", "boolean"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "annotations", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "details": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "label": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "events", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "server_received_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "app": { + "type": ["null", "integer"] + }, + "device_carrier": { + "type": ["null", "string"] + }, + "$schema": { + "type": ["null", "integer"] + }, + "city": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "string"] + }, + "uuid": { + "type": ["null", "string"] + }, + "event_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "platform": { + "type": ["null", "string"] + }, + "os_version": { + "type": ["null", "string"] + }, + "amplitude_id": { + "type": ["null", "integer"] + }, + "processed_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "user_creation_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "version_name": { + "type": ["null", "string"] + }, + "ip_address": { + "type": ["null", "string"] + }, + "paying": { + "type": ["null", "boolean"] + }, + "dma": { + "type": ["null", "string"] + }, + "group_properties": { + "type": ["null", "object"] + }, + "user_properties": { + "type": ["null", "object"] + }, + "client_upload_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "$insert_id": { + "type": ["null", "string"] + }, + "event_type": { + "type": ["null", "string"] + }, + "library": { + "type": ["null", "string"] + }, + "amplitude_attribution_ids": { + "type": ["null", "string"] + }, + "device_type": { + "type": ["null", "string"] + }, + "device_manufacturer": { + "type": ["null", "string"] + }, + "start_version": { + "type": ["null", "string"] + }, + "location_lng": { + "type": ["null", "number"] + }, + "server_upload_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "event_id": { + "type": ["null", "integer"] + }, + "location_lat": { + "type": ["null", "number"] + }, + "os_name": { + "type": ["null", "string"] + }, + "amplitude_event_type": { + "type": ["null", "string"] + }, + "device_brand": { + "type": ["null", "string"] + }, + "groups": { + "type": ["null", "object"] + }, + "event_properties": { + "type": ["null", "object"] + }, + "data": { + "type": ["null", "object"] + }, + "device_id": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + }, + "device_model": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "is_attribution_event": { + "type": ["null", "boolean"] + }, + "adid": { + "type": ["null", "string"] + }, + "session_id": { + "type": ["null", "number"] + }, + "device_family": { + "type": ["null", "string"] + }, + "sample_rate": { + "type": ["null"] + }, + "idfa": { + "type": ["null", "string"] + }, + "client_event_time": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["event_time"], + "source_defined_primary_key": [["uuid"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["event_time"] + }, + { + "stream": { + "name": "active_users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "statistics": { + "type": ["null", "object"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["date"] + }, + { + "stream": { + "name": "average_session_length", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "length": { + "type": ["null", "number"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["date"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog_without_events.json b/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog_without_events.json new file mode 100644 index 000000000000..c95535c29a94 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/configured_catalog_without_events.json @@ -0,0 +1,145 @@ +{ + "streams": [ + { + "stream": { + "name": "cohorts", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "appId": { + "type": ["null", "integer"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "definition": { + "type": ["null", "object"] + }, + "description": { + "type": ["null", "string"] + }, + "finished": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "owners": { + "type": ["null", "object"] + }, + "published": { + "type": ["null", "boolean"] + }, + "size": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "lastMod": { + "type": ["null", "integer"] + }, + "lastComputed": { + "type": ["null", "integer"] + }, + "hidden": { + "type": ["null", "boolean"] + }, + "is_predictive": { + "type": ["null", "boolean"] + }, + "is_official_content": { + "type": ["null", "boolean"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "annotations", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "details": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "label": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "active_users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "statistics": { + "type": ["null", "object"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["date"] + }, + { + "stream": { + "name": "average_session_length", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "length": { + "type": ["null", "number"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["date"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-amplitude/integration_tests/invalid_config.json new file mode 100644 index 000000000000..37e6d1564cab --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "", + "secret_key": "", + "start_date": "2021-05-25T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-amplitude/integration_tests/sample_config.json new file mode 100644 index 000000000000..629cbd8815f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "", + "secret_key": "", + "start_date": "2021-05-25T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-amplitude/integration_tests/sample_state.json new file mode 100644 index 000000000000..cb907b53c2b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/sample_state.json @@ -0,0 +1,11 @@ +{ + "events": { + "event_time": "2021-05-27 11:59:53.710000" + }, + "active_users": { + "date": "2021-05-27" + }, + "average_session_length": { + "date": "2021-05-27" + } +} diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/streams_with_output_records_catalog.json b/airbyte-integrations/connectors/source-amplitude/integration_tests/streams_with_output_records_catalog.json new file mode 100644 index 000000000000..e6a38a6ca212 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/streams_with_output_records_catalog.json @@ -0,0 +1,117 @@ +{ + "streams": [ + { + "stream": { + "name": "cohorts", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "appId": { + "type": ["null", "integer"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "definition": { + "type": ["null", "object"] + }, + "description": { + "type": ["null", "string"] + }, + "finished": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "owners": { + "type": ["null", "object"] + }, + "published": { + "type": ["null", "boolean"] + }, + "size": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "lastMod": { + "type": ["null", "integer"] + }, + "lastComputed": { + "type": ["null", "integer"] + }, + "hidden": { + "type": ["null", "boolean"] + }, + "is_predictive": { + "type": ["null", "boolean"] + }, + "is_official_content": { + "type": ["null", "boolean"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "active_users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "statistics": { + "type": ["null", "object"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["date"] + }, + { + "stream": { + "name": "average_session_length", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "length": { + "type": ["null", "number"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["date"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-amplitude/main.py b/airbyte-integrations/connectors/source-amplitude/main.py new file mode 100644 index 000000000000..6a7f9481cf1e --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_amplitude import SourceAmplitude + +if __name__ == "__main__": + source = SourceAmplitude() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amplitude/setup.py b/airbyte-integrations/connectors/source-amplitude/setup.py new file mode 100644 index 000000000000..49f5689204dd --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/setup.py @@ -0,0 +1,45 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = ["pytest~=6.1"] + +setup( + name="source_amplitude", + description="Source implementation for Amplitude.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/__init__.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/__init__.py new file mode 100644 index 000000000000..2c519b869478 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/__init__.py @@ -0,0 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from .source import SourceAmplitude + +__all__ = ["SourceAmplitude"] diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py new file mode 100644 index 000000000000..3c3e3e67056d --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/api.py @@ -0,0 +1,209 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import gzip +import io +import json +import urllib.parse as urlparse +import zipfile +from abc import ABC, abstractmethod +from typing import IO, Any, Iterable, List, Mapping, MutableMapping, Optional +from urllib.parse import parse_qs + +import pendulum +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http import HttpStream + + +class AmplitudeStream(HttpStream, ABC): + + url_base = "https://amplitude.com/api" + api_version = 2 + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + respose_data = response.json() + yield from respose_data.get(self.name, []) + + def path(self, **kwargs) -> str: + return f"/{self.api_version}/{self.name}" + + +class Cohorts(AmplitudeStream): + primary_key = "id" + api_version = 3 + + +class Annotations(AmplitudeStream): + primary_key = "id" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + respose_data = response.json() + yield from respose_data.get("data", []) + + +class IncrementalAmplitudeStream(AmplitudeStream, ABC): + state_checkpoint_interval = 10 + base_params = {} + cursor_field = "date" + date_template = "%Y%m%d" + + def __init__(self, start_date: str, **kwargs): + super().__init__(**kwargs) + self._start_date = pendulum.parse(start_date) + + @property + @abstractmethod + def time_interval(self) -> dict: + """ + Defining the time interval to determine the difference between the end date and the start date. + """ + pass + + def _get_end_date(self, current_date: pendulum, end_date: pendulum = pendulum.now()): + if current_date.add(**self.time_interval).date() < end_date.date(): + end_date = current_date.add(**self.time_interval) + return end_date + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + latest_benchmark = latest_record[self.cursor_field] + if current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} + return {self.cursor_field: latest_benchmark} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + parsed = urlparse.urlparse(response.url) + end = parse_qs(parsed.query).get("end", None) + if end: + end_time = pendulum.parse(end[0]) + now = pendulum.now() + if end_time.date() < now.date(): + return {"start": end[0], "end": self._get_end_date(end_time).strftime(self.date_template)} + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = self.base_params + if next_page_token: + params.update(next_page_token) + else: + start_datetime = self._start_date + if stream_state.get(self.cursor_field): + start_datetime = pendulum.parse(stream_state[self.cursor_field]) + + params.update( + { + "start": start_datetime.strftime(self.date_template), + "end": self._get_end_date(start_datetime).strftime(self.date_template), + } + ) + return params + + +class Events(IncrementalAmplitudeStream): + cursor_field = "event_time" + date_template = "%Y%m%dT%H" + primary_key = "uuid" + state_checkpoint_interval = 1000 + time_interval = {"days": 3} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + zip_file = zipfile.ZipFile(io.BytesIO(response.content)) + for gzip_filename in zip_file.namelist(): + with zip_file.open(gzip_filename) as file: + yield from self._parse_zip_file(file) + + def _parse_zip_file(self, zip_file: IO[bytes]) -> Iterable[Mapping]: + with gzip.open(zip_file) as file: + for record in file: + yield json.loads(record) + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + stream_state = stream_state or {} + params = self.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=None) + + # API returns data only when requested with a difference between 'start' and 'end' of 6 or more hours. + if pendulum.parse(params["start"]).add(hours=6) > pendulum.parse(params["end"]): + return [] + yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + + def request_params( + self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) + if stream_state or next_page_token: + params["start"] = pendulum.parse(params["start"]).add(hours=1).strftime(self.date_template) + return params + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return f"/{self.api_version}/export" + + +class ActiveUsers(IncrementalAmplitudeStream): + base_params = {"m": "active", "i": 1, "g": "country"} + name = "active_users" + primary_key = "date" + time_interval = {"months": 1} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_data = response.json().get("data", []) + if response_data: + series = list(map(list, zip(*response_data["series"]))) + for i, date in enumerate(response_data["xValues"]): + yield {"date": date, "statistics": dict(zip(response_data["seriesLabels"], series[i]))} + + def path(self, **kwargs) -> str: + return f"/{self.api_version}/users" + + +class AverageSessionLength(IncrementalAmplitudeStream): + name = "average_session_length" + primary_key = "date" + time_interval = {"days": 15} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_data = response.json().get("data", []) + if response_data: + # From the Amplitude documentation it follows that "series" is an array with one element which is itself + # an array that contains the average session length for each day. + # https://developers.amplitude.com/docs/dashboard-rest-api#returns-2 + series = response_data["series"][0] + for i, date in enumerate(response_data["xValues"]): + yield {"date": date, "length": series[i]} + + def path(self, **kwargs) -> str: + return f"/{self.api_version}/sessions/average" diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/active_users.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/active_users.json new file mode 100644 index 000000000000..f8707f899b84 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/active_users.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "statistics": { + "type": ["null", "object"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/annotations.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/annotations.json new file mode 100644 index 000000000000..012beba10fe5 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/annotations.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "details": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "label": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json new file mode 100644 index 000000000000..6d52f4e13510 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["null", "string"], + "format": "date-time" + }, + "length": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/cohorts.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/cohorts.json new file mode 100644 index 000000000000..71bd4520bcb5 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/cohorts.json @@ -0,0 +1,54 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "appId": { + "type": ["null", "integer"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "definition": { + "type": ["null", "object"] + }, + "description": { + "type": ["null", "string"] + }, + "finished": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "owners": { + "type": ["null", "object"] + }, + "published": { + "type": ["null", "boolean"] + }, + "size": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "lastMod": { + "type": ["null", "integer"] + }, + "lastComputed": { + "type": ["null", "integer"] + }, + "hidden": { + "type": ["null", "boolean"] + }, + "is_predictive": { + "type": ["null", "boolean"] + }, + "is_official_content": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/events.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/events.json new file mode 100644 index 000000000000..6638e22de4f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/events.json @@ -0,0 +1,160 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "server_received_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "app": { + "type": ["null", "integer"] + }, + "device_carrier": { + "type": ["null", "string"] + }, + "$schema": { + "type": ["null", "integer"] + }, + "city": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "string"] + }, + "uuid": { + "type": ["null", "string"] + }, + "event_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "platform": { + "type": ["null", "string"] + }, + "os_version": { + "type": ["null", "string"] + }, + "amplitude_id": { + "type": ["null", "integer"] + }, + "processed_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "user_creation_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "version_name": { + "type": ["null", "string"] + }, + "ip_address": { + "type": ["null", "string"] + }, + "paying": { + "type": ["null", "boolean"] + }, + "dma": { + "type": ["null", "string"] + }, + "group_properties": { + "type": ["null", "object"] + }, + "user_properties": { + "type": ["null", "object"] + }, + "client_upload_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "$insert_id": { + "type": ["null", "string"] + }, + "event_type": { + "type": ["null", "string"] + }, + "library": { + "type": ["null", "string"] + }, + "amplitude_attribution_ids": { + "type": ["null", "string"] + }, + "device_type": { + "type": ["null", "string"] + }, + "device_manufacturer": { + "type": ["null", "string"] + }, + "start_version": { + "type": ["null", "string"] + }, + "location_lng": { + "type": ["null", "number"] + }, + "server_upload_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "event_id": { + "type": ["null", "integer"] + }, + "location_lat": { + "type": ["null", "number"] + }, + "os_name": { + "type": ["null", "string"] + }, + "amplitude_event_type": { + "type": ["null", "string"] + }, + "device_brand": { + "type": ["null", "string"] + }, + "groups": { + "type": ["null", "object"] + }, + "event_properties": { + "type": ["null", "object"] + }, + "data": { + "type": ["null", "object"] + }, + "device_id": { + "type": ["null", "string"] + }, + "language": { + "type": ["null", "string"] + }, + "device_model": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "is_attribution_event": { + "type": ["null", "boolean"] + }, + "adid": { + "type": ["null", "string"] + }, + "session_id": { + "type": ["null", "number"] + }, + "device_family": { + "type": ["null", "string"] + }, + "sample_rate": { + "type": ["null"] + }, + "idfa": { + "type": ["null", "string"] + }, + "client_event_time": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py new file mode 100644 index 000000000000..42e73b5addd6 --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py @@ -0,0 +1,64 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from base64 import b64encode +from typing import Any, List, Mapping, Tuple + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +from .api import ActiveUsers, Annotations, AverageSessionLength, Cohorts, Events + + +class SourceAmplitude(AbstractSource): + def _convert_auth_to_token(self, username: str, password: str) -> str: + username = username.encode("latin1") + password = password.encode("latin1") + token = b64encode(b":".join((username, password))).strip().decode("ascii") + return token + + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: + try: + auth = TokenAuthenticator(token=self._convert_auth_to_token(config["api_key"], config["secret_key"]), auth_method="Basic") + list(Cohorts(authenticator=auth).read_records(SyncMode.full_refresh)) + return True, None + except Exception as error: + return False, f"Unable to connect to Amplitude API with the provided credentials - {repr(error)}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + + auth = TokenAuthenticator(token=self._convert_auth_to_token(config["api_key"], config["secret_key"]), auth_method="Basic") + return [ + Cohorts(authenticator=auth), + Annotations(authenticator=auth), + Events(authenticator=auth, start_date=config["start_date"]), + ActiveUsers(authenticator=auth, start_date=config["start_date"]), + AverageSessionLength(authenticator=auth, start_date=config["start_date"]), + ] diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.json new file mode 100644 index 000000000000..f8bdc70c05ec --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.json @@ -0,0 +1,28 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/amplitude", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Amplitude Spec", + "type": "object", + "required": ["api_key", "secret_key", "start_date"], + "additionalProperties": false, + "properties": { + "api_key": { + "type": "string", + "description": "This is the project’s API key, used for calling Amplitude’s APIs", + "airbyte_secret": true + }, + "secret_key": { + "type": "string", + "description": "This is the project's secret key, which is also used for calling Amplitude’s APIs", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "description": "UTC date and time in the format 2021-01-25T00:00:00Z. Any data before this date will not be replicated.", + "examples": ["2021-01-25T00:00:00Z"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-amplitude/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-amplitude/unit_tests/unit_test.py new file mode 100644 index 000000000000..b8a8150b507f --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/unit_tests/unit_test.py @@ -0,0 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +def test_example_method(): + assert True diff --git a/airbyte-integrations/connectors/source-harvest/.dockerignore b/airbyte-integrations/connectors/source-harvest/.dockerignore new file mode 100644 index 000000000000..7f4027ee9cb3 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/.dockerignore @@ -0,0 +1,5 @@ +* +!main.py +!source_harvest +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-harvest/Dockerfile b/airbyte-integrations/connectors/source-harvest/Dockerfile new file mode 100644 index 000000000000..64582a00fbdf --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_harvest ./source_harvest +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-harvest diff --git a/airbyte-integrations/connectors/source-harvest/README.md b/airbyte-integrations/connectors/source-harvest/README.md new file mode 100644 index 000000000000..87f00fa3a467 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/README.md @@ -0,0 +1,129 @@ +# Harvest Source + +This is the repository for the Harvest source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/harvest). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-harvest:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/harvest) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_harvest/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source harvest test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-harvest:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-harvest:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-harvest:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-harvest:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-harvest:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-harvest:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-harvest:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-harvest:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml b/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml new file mode 100644 index 000000000000..8c47e03bb458 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml @@ -0,0 +1,25 @@ +connector_image: airbyte/source-harvest:dev +tests: + spec: + - spec_path: "source_harvest/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + cursor_paths: + contacts: ["updated_at"] + expenses_clients: ["to"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-harvest/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-harvest/acceptance-test-docker.sh new file mode 100755 index 000000000000..1425ff74f151 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-harvest/build.gradle b/airbyte-integrations/connectors/source-harvest/build.gradle new file mode 100644 index 000000000000..b2315be59888 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_harvest' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/__init__.py b/airbyte-integrations/connectors/source-harvest/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-harvest/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..d5687c658eaf --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/abnormal_state.json @@ -0,0 +1,86 @@ +{ + "contacts": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "clients": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "invoices": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "invoice_messages": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "invoice_payments": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "invoice_item_categories": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "estimates": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "estimate_messages": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "estimate_item_categories": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "expenses": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "expense_categories": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "tasks": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "time_entries": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "user_assignments": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "task_assignments": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "projects": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "roles": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "users": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "project_assignments": { + "updated_at": "2217-06-26T21:20:07Z" + }, + "expenses_clients": { + "to": "22170626" + }, + "expenses_projects": { + "to": "22170626" + }, + "expenses_categories": { + "to": "22170626" + }, + "expenses_team": { + "to": "22170626" + }, + "uninvoiced": { + "to": "22170626" + }, + "time_clients": { + "to": "22170626" + }, + "time_projects": { + "to": "22170626" + }, + "time_tasks": { + "to": "22170626" + }, + "time_team": { + "to": "22170626" + } +} diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py new file mode 100644 index 000000000000..eeb4a2d3e02e --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-harvest/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..1c8f2675fffb --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/configured_catalog.json @@ -0,0 +1,2060 @@ +{ + "streams": [ + { + "stream": { + "name": "clients", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "address": { + "type": ["null", "string"] + }, + "statement_key": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "currency": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "contacts", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone_office": { + "type": ["null", "string"] + }, + "phone_mobile": { + "type": ["null", "string"] + }, + "fax": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "company", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "base_uri": { + "type": ["null", "string"] + }, + "full_domain": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "week_start_day": { + "type": ["null", "string"] + }, + "wants_timestamp_timers": { + "type": ["null", "boolean"] + }, + "time_format": { + "type": ["null", "string"] + }, + "plan_type": { + "type": ["null", "string"] + }, + "expense_feature": { + "type": ["null", "boolean"] + }, + "invoice_feature": { + "type": ["null", "boolean"] + }, + "estimate_feature": { + "type": ["null", "boolean"] + }, + "approval_required": { + "type": ["null", "boolean"] + }, + "clock": { + "type": ["null", "string"] + }, + "decimal_symbol": { + "type": ["null", "string"] + }, + "thousands_separator": { + "type": ["null", "string"] + }, + "color_scheme": { + "type": ["null", "string"] + }, + "weekly_capacity": { + "type": ["null", "integer"] + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "invoices", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "client_key": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "string"] + }, + "purchase_order": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"] + }, + "due_amount": { + "type": ["null", "number"] + }, + "tax": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"] + }, + "tax2": { + "type": ["null", "integer"] + }, + "tax2_amount": { + "type": ["null", "number"] + }, + "discount": { + "type": ["null", "integer"] + }, + "discount_amount": { + "type": ["null", "integer"] + }, + "subject": { + "type": ["null", "string"] + }, + "notes": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "period_start": { + "type": ["null", "string"], + "format": "date" + }, + "period_end": { + "type": ["null", "string"], + "format": "date" + }, + "issue_date": { + "type": ["null", "string"], + "format": "date" + }, + "due_date": { + "type": ["null", "string"], + "format": "date" + }, + "payment_term": { + "type": ["null", "string"] + }, + "sent_at": { + "type": ["null", "string"], + "format": "date" + }, + "paid_at": { + "type": ["null", "string"] + }, + "paid_date": { + "type": ["null", "string"] + }, + "closed_at": { + "type": ["null", "string"] + }, + "recurring_invoice_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date" + }, + "currency": { + "type": ["null", "string"] + }, + "client": { + "type": ["null", "string"] + }, + "estimate": { + "type": ["null", "string"] + }, + "retainer": { + "type": ["null", "string"] + }, + "creator": { + "type": ["null", "string"] + }, + "line_items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "kind": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "unit_price": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "integer"] + }, + "taxed": { + "type": ["null", "boolean"] + }, + "taxed2": { + "type": ["null", "boolean"] + }, + "project": { + "type": ["null", "string"] + } + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "invoice_messages", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "sent_by": { + "type": ["null", "string"] + }, + "sent_by_email": { + "type": ["null", "string"] + }, + "sent_from": { + "type": ["null", "string"] + }, + "sent_from_email": { + "type": ["null", "string"] + }, + "include_link_to_client_invoice": { + "type": ["null", "boolean"] + }, + "send_me_a_copy": { + "type": ["null", "boolean"] + }, + "thank_you": { + "type": ["null", "boolean"] + }, + "reminder": { + "type": ["null", "boolean"] + }, + "send_reminder_on": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date" + }, + "attach_pdf": { + "type": ["null", "boolean"] + }, + "event_type": { + "type": ["null", "string"] + }, + "recipients": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + } + }, + "subject": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "invoice_payments", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "integer"] + }, + "paid_at": { + "type": ["null", "string"], + "format": "date" + }, + "paid_date": { + "type": ["null", "string"], + "format": "date" + }, + "recorded_by": { + "type": ["null", "string"] + }, + "recorded_by_email": { + "type": ["null", "string"] + }, + "notes": { + "type": ["null", "string"] + }, + "transaction_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date" + }, + "payment_gateway": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "invoice_item_categories", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "use_as_service": { + "type": ["null", "boolean"] + }, + "use_as_expense": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "estimates", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "client_key": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "string"] + }, + "purchase_order": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"] + }, + "tax": { + "type": ["null", "number"] + }, + "tax_amount": { + "type": ["null", "number"] + }, + "tax2": { + "type": ["null", "number"] + }, + "tax2_amount": { + "type": ["null", "number"] + }, + "discount": { + "type": ["null", "number"] + }, + "discount_amount": { + "type": ["null", "number"] + }, + "subject": { + "type": ["null", "string"] + }, + "notes": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "issue_date": { + "type": ["null", "string"], + "format": "date" + }, + "sent_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "accepted_at": { + "type": ["null", "string"] + }, + "declined_at": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "creator": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "line_items": { + "type": ["null", "array"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "estimate_messages", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "sent_by": { + "type": ["null", "string"] + }, + "sent_by_email": { + "type": ["null", "string"] + }, + "sent_from": { + "type": ["null", "string"] + }, + "sent_from_email": { + "type": ["null", "string"] + }, + "send_me_a_copy": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "recipients": { + "type": ["null", "array"] + }, + "event_type": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "estimate_item_categories", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "expenses", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "notes": { + "type": ["null", "string"] + }, + "total_cost": { + "type": ["null", "number"] + }, + "units": { + "type": ["null", "number"] + }, + "is_closed": { + "type": ["null", "boolean"] + }, + "is_locked": { + "type": ["null", "boolean"] + }, + "is_billed": { + "type": ["null", "boolean"] + }, + "locked_reason": { + "type": ["null", "string"] + }, + "spent_date": { + "type": ["null", "string"], + "format": "date" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "billable": { + "type": ["null", "boolean"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "file_name": { + "type": ["null", "string"] + }, + "file_size": { + "type": ["null", "integer"] + }, + "content_type": { + "type": ["null", "string"] + } + } + }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "user_assignment": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "is_project_manager": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "budget": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + } + } + }, + "project": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + } + } + }, + "expense_category": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "unit_price": { + "type": ["null", "string"] + }, + "unit_name": { + "type": ["null", "string"] + } + } + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + } + } + }, + "invoice": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "number": { + "type": ["null", "string"] + } + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "expense_categories", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "unit_name": { + "type": ["null", "string"] + }, + "unit_price": { + "type": ["null", "number"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "tasks", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "billable_by_default": { + "type": ["null", "boolean"] + }, + "default_hourly_rate": { + "type": ["null", "number"] + }, + "is_default": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "time_entries", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "spent_date": { + "type": ["null", "string"], + "format": "date" + }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "project": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "task": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "user_assignment": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "is_project_manager": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "budget": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + } + } + }, + "task_assignment": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "billable": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + }, + "budget": { + "type": ["null", "string"] + } + } + }, + "hours": { + "type": ["null", "number"] + }, + "rounded_hours": { + "type": ["null", "number"] + }, + "notes": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "is_locked": { + "type": ["null", "boolean"] + }, + "locked_reason": { + "type": ["null", "string"] + }, + "is_closed": { + "type": ["null", "boolean"] + }, + "is_billed": { + "type": ["null", "boolean"] + }, + "timer_started_at": { + "type": ["null", "string"] + }, + "started_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "ended_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "is_running": { + "type": ["null", "boolean"] + }, + "invoice": { + "type": ["null", "string"] + }, + "external_reference": { + "type": ["null", "string"] + }, + "billable": { + "type": ["null", "boolean"] + }, + "budgeted": { + "type": ["null", "boolean"] + }, + "billable_rate": { + "type": ["null", "number"] + }, + "cost_rate": { + "type": ["null", "number"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "user_assignments", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "is_project_manager": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "use_default_rates": { + "type": ["null", "boolean"] + }, + "budget": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + }, + "project": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + } + } + }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "task_assignments", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "billable": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + }, + "budget": { + "type": ["null", "string"] + }, + "project": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + } + } + }, + "task": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "projects", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "bill_by": { + "type": ["null", "string"] + }, + "budget": { + "type": ["null", "number"] + }, + "budget_by": { + "type": ["null", "string"] + }, + "budget_is_monthly": { + "type": ["null", "boolean"] + }, + "notify_when_over_budget": { + "type": ["null", "boolean"] + }, + "over_budget_notification_percentage": { + "type": ["null", "number"] + }, + "over_budget_notification_date": { + "type": ["null", "string"], + "format": "date" + }, + "show_budget_to_all": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "starts_on": { + "type": ["null", "string"], + "format": "date" + }, + "ends_on": { + "type": ["null", "string"] + }, + "is_billable": { + "type": ["null", "boolean"] + }, + "is_fixed_fee": { + "type": ["null", "boolean"] + }, + "notes": { + "type": ["null", "string"] + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + } + } + }, + "cost_budget": { + "type": ["null", "string"] + }, + "cost_budget_include_expenses": { + "type": ["null", "boolean"] + }, + "hourly_rate": { + "type": ["null", "number"] + }, + "fee": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "roles", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "user_ids": { + "type": ["null", "array"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "telephone": { + "type": ["null", "string"] + }, + "timezone": { + "type": ["null", "string"] + }, + "has_access_to_all_future_projects": { + "type": ["null", "boolean"] + }, + "is_contractor": { + "type": ["null", "boolean"] + }, + "is_admin": { + "type": ["null", "boolean"] + }, + "is_project_manager": { + "type": ["null", "boolean"] + }, + "can_see_rates": { + "type": ["null", "boolean"] + }, + "can_create_projects": { + "type": ["null", "boolean"] + }, + "can_create_invoices": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "weekly_capacity": { + "type": ["null", "integer"] + }, + "default_hourly_rate": { + "type": ["null", "number"] + }, + "cost_rate": { + "type": ["null", "number"] + }, + "roles": { + "type": ["null", "array"] + }, + "avatar_url": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "billable_rates", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date" + }, + "end_date": { + "type": ["null", "string"], + "format": "date" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "cost_rates", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date" + }, + "end_date": { + "type": ["null", "string"], + "format": "date" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "project_assignments", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "is_project_manager": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "use_default_rates": { + "type": ["null", "boolean"] + }, + "budget": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + }, + "project": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + } + } + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "task_assignments": { + "type": ["null", "array"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "expenses_clients", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "total_amount": { + "type": ["null", "integer"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["to"] + }, + "sync_mode": "incremental", + "cursor_field": ["to"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "expenses_projects", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "project_id": { + "type": ["null", "integer"] + }, + "project_name": { + "type": ["null", "string"] + }, + "total_amount": { + "type": ["null", "number"] + }, + "billable_amount": { + "type": ["null", "number"] + }, + "currency": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["to"] + }, + "sync_mode": "incremental", + "cursor_field": ["to"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "expenses_categories", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "expense_category_id": { + "type": ["null", "integer"] + }, + "expense_category_name": { + "type": ["null", "string"] + }, + "total_amount": { + "type": ["null", "integer"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["to"] + }, + "sync_mode": "incremental", + "cursor_field": ["to"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "expenses_team", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "user_id": { + "type": ["null", "integer"] + }, + "user_name": { + "type": ["null", "string"] + }, + "is_contractor": { + "type": ["null", "boolean"] + }, + "total_amount": { + "type": ["null", "integer"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["to"] + }, + "sync_mode": "incremental", + "cursor_field": ["to"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "uninvoiced", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "project_id": { + "type": ["null", "integer"] + }, + "project_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "total_hours": { + "type": ["null", "integer"] + }, + "uninvoiced_hours": { + "type": ["null", "integer"] + }, + "uninvoiced_expenses": { + "type": ["null", "integer"] + }, + "uninvoiced_amount": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "time_clients", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "total_hours": { + "type": ["null", "number"] + }, + "billable_hours": { + "type": ["null", "number"] + }, + "currency": { + "type": ["null", "string"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["to"] + }, + "sync_mode": "incremental", + "cursor_field": ["to"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "time_projects", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "project_id": { + "type": ["null", "integer"] + }, + "project_name": { + "type": ["null", "string"] + }, + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "total_hours": { + "type": ["null", "integer"] + }, + "billable_hours": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["to"] + }, + "sync_mode": "incremental", + "cursor_field": ["to"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "time_tasks", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "task_id": { + "type": ["null", "integer"] + }, + "task_name": { + "type": ["null", "string"] + }, + "total_hours": { + "type": ["null", "integer"] + }, + "billable_hours": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["to"] + }, + "sync_mode": "incremental", + "cursor_field": ["to"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "time_team", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "user_id": { + "type": ["null", "integer"] + }, + "user_name": { + "type": ["null", "string"] + }, + "is_contractor": { + "type": ["null", "boolean"] + }, + "total_hours": { + "type": ["null", "number"] + }, + "billable_hours": { + "type": ["null", "number"] + }, + "currency": { + "type": ["null", "string"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["to"] + }, + "sync_mode": "incremental", + "cursor_field": ["to"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "project_budget", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "project_id": { + "type": ["null", "integer"] + }, + "project_name": { + "type": ["null", "string"] + }, + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "budget_is_monthly": { + "type": ["null", "boolean"] + }, + "budget_by": { + "type": ["null", "string"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "budget": { + "type": ["null", "integer"] + }, + "budget_spent": { + "type": ["null", "integer"] + }, + "budget_remaining": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-harvest/integration_tests/invalid_config.json new file mode 100644 index 000000000000..57b07556f3ff --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "api_token": "1111111.aa.wrong-api-token", + "account_id": "1111111", + "replication_start_date": "1000-06-26T21:20:07Z" +} diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-harvest/integration_tests/sample_config.json new file mode 100644 index 000000000000..a9d85c28758b --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_token": "", + "account_id": "", + "replication_start_date": "2021-01-01T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-harvest/integration_tests/sample_state.json new file mode 100644 index 000000000000..ae4bb37a212e --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/sample_state.json @@ -0,0 +1,86 @@ +{ + "contacts": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "clients": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "invoices": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "invoice_messages": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "invoice_payments": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "invoice_item_categories": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "estimates": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "estimate_messages": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "estimate_item_categories": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "expenses": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "expense_categories": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "tasks": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "time_entries": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "user_assignments": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "task_assignments": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "projects": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "roles": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "users": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "project_assignments": { + "updated_at": "2021-01-01T00:00:00Z" + }, + "expenses_clients": { + "to": "20210101" + }, + "expenses_projects": { + "to": "20210101" + }, + "expenses_categories": { + "to": "20210101" + }, + "expenses_team": { + "to": "20210101" + }, + "uninvoiced": { + "to": "20210101" + }, + "time_clients": { + "to": "20210101" + }, + "time_projects": { + "to": "20210101" + }, + "time_tasks": { + "to": "20210101" + }, + "time_team": { + "to": "20210101" + } +} diff --git a/airbyte-integrations/connectors/source-harvest/main.py b/airbyte-integrations/connectors/source-harvest/main.py new file mode 100644 index 000000000000..24cbed7bada4 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_harvest import SourceHarvest + +if __name__ == "__main__": + source = SourceHarvest() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-harvest/setup.py b/airbyte-integrations/connectors/source-harvest/setup.py new file mode 100644 index 000000000000..681e8e178fd9 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/setup.py @@ -0,0 +1,47 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", +] + +setup( + name="source_harvest", + description="Source implementation for Harvest.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/__init__.py b/airbyte-integrations/connectors/source-harvest/source_harvest/__init__.py new file mode 100644 index 000000000000..0558fdc23880 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourceHarvest + +__all__ = ["SourceHarvest"] diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py b/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py new file mode 100644 index 000000000000..5a654fd0a1e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/auth.py @@ -0,0 +1,37 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from typing import Any, Mapping + +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + + +class HarvestTokenAuthenticator(TokenAuthenticator): + def __init__(self, token: str, account_id: str, account_id_header: str = "Harvest-Account-ID", **kwargs): + super().__init__(token, **kwargs) + self.account_id = account_id + self.account_id_header = account_id_header + + def get_auth_header(self) -> Mapping[str, Any]: + return {**super().get_auth_header(), self.account_id_header: self.account_id} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/billable_rates.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/billable_rates.json new file mode 100644 index 000000000000..cc071925d0ba --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/billable_rates.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date" + }, + "end_date": { + "type": ["null", "string"], + "format": "date" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/clients.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/clients.json new file mode 100644 index 000000000000..050e85d13bf2 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/clients.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "address": { + "type": ["null", "string"] + }, + "statement_key": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "currency": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/company.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/company.json new file mode 100644 index 000000000000..704f3d7cb62a --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/company.json @@ -0,0 +1,57 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "base_uri": { + "type": ["null", "string"] + }, + "full_domain": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "week_start_day": { + "type": ["null", "string"] + }, + "wants_timestamp_timers": { + "type": ["null", "boolean"] + }, + "time_format": { + "type": ["null", "string"] + }, + "plan_type": { + "type": ["null", "string"] + }, + "expense_feature": { + "type": ["null", "boolean"] + }, + "invoice_feature": { + "type": ["null", "boolean"] + }, + "estimate_feature": { + "type": ["null", "boolean"] + }, + "approval_required": { + "type": ["null", "boolean"] + }, + "clock": { + "type": ["null", "string"] + }, + "decimal_symbol": { + "type": ["null", "string"] + }, + "thousands_separator": { + "type": ["null", "string"] + }, + "color_scheme": { + "type": ["null", "string"] + }, + "weekly_capacity": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/contacts.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/contacts.json new file mode 100644 index 000000000000..272d6f40a36e --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/contacts.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone_office": { + "type": ["null", "string"] + }, + "phone_mobile": { + "type": ["null", "string"] + }, + "fax": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/cost_rates.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/cost_rates.json new file mode 100644 index 000000000000..cc071925d0ba --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/cost_rates.json @@ -0,0 +1,28 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date" + }, + "end_date": { + "type": ["null", "string"], + "format": "date" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_item_categories.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_item_categories.json new file mode 100644 index 000000000000..0c6b5ce3dd73 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_item_categories.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_messages.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_messages.json new file mode 100644 index 000000000000..2157406a16be --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimate_messages.json @@ -0,0 +1,44 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "sent_by": { + "type": ["null", "string"] + }, + "sent_by_email": { + "type": ["null", "string"] + }, + "sent_from": { + "type": ["null", "string"] + }, + "sent_from_email": { + "type": ["null", "string"] + }, + "send_me_a_copy": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "recipients": { + "type": ["null", "array"] + }, + "event_type": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimates.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimates.json new file mode 100644 index 000000000000..3d995ca072d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/estimates.json @@ -0,0 +1,98 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "client_key": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "string"] + }, + "purchase_order": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"] + }, + "tax": { + "type": ["null", "number"] + }, + "tax_amount": { + "type": ["null", "number"] + }, + "tax2": { + "type": ["null", "number"] + }, + "tax2_amount": { + "type": ["null", "number"] + }, + "discount": { + "type": ["null", "number"] + }, + "discount_amount": { + "type": ["null", "number"] + }, + "subject": { + "type": ["null", "string"] + }, + "notes": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "issue_date": { + "type": ["null", "string"], + "format": "date" + }, + "sent_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "accepted_at": { + "type": ["null", "string"] + }, + "declined_at": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "creator": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "line_items": { + "type": ["null", "array"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expense_categories.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expense_categories.json new file mode 100644 index 000000000000..332043c1dc24 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expense_categories.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "unit_name": { + "type": ["null", "string"] + }, + "unit_price": { + "type": ["null", "number"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses.json new file mode 100644 index 000000000000..547c6c54ce9d --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses.json @@ -0,0 +1,157 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "notes": { + "type": ["null", "string"] + }, + "total_cost": { + "type": ["null", "number"] + }, + "units": { + "type": ["null", "number"] + }, + "is_closed": { + "type": ["null", "boolean"] + }, + "is_locked": { + "type": ["null", "boolean"] + }, + "is_billed": { + "type": ["null", "boolean"] + }, + "locked_reason": { + "type": ["null", "string"] + }, + "spent_date": { + "type": ["null", "string"], + "format": "date" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "billable": { + "type": ["null", "boolean"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "file_name": { + "type": ["null", "string"] + }, + "file_size": { + "type": ["null", "integer"] + }, + "content_type": { + "type": ["null", "string"] + } + } + }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "user_assignment": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "is_project_manager": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "budget": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + } + } + }, + "project": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + } + } + }, + "expense_category": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "unit_price": { + "type": ["null", "string"] + }, + "unit_name": { + "type": ["null", "string"] + } + } + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + } + } + }, + "invoice": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "number": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_categories.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_categories.json new file mode 100644 index 000000000000..8a3f3503b69d --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_categories.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "expense_category_id": { + "type": ["null", "integer"] + }, + "expense_category_name": { + "type": ["null", "string"] + }, + "total_amount": { + "type": ["null", "integer"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_clients.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_clients.json new file mode 100644 index 000000000000..c101189f93a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_clients.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "total_amount": { + "type": ["null", "integer"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_projects.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_projects.json new file mode 100644 index 000000000000..23d2ee0556bf --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_projects.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "project_id": { + "type": ["null", "integer"] + }, + "project_name": { + "type": ["null", "string"] + }, + "total_amount": { + "type": ["null", "number"] + }, + "billable_amount": { + "type": ["null", "number"] + }, + "currency": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_team.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_team.json new file mode 100644 index 000000000000..e107c3f12509 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/expenses_team.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "user_id": { + "type": ["null", "integer"] + }, + "user_name": { + "type": ["null", "string"] + }, + "is_contractor": { + "type": ["null", "boolean"] + }, + "total_amount": { + "type": ["null", "integer"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_item_categories.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_item_categories.json new file mode 100644 index 000000000000..4ce497f49c51 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_item_categories.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "use_as_service": { + "type": ["null", "boolean"] + }, + "use_as_expense": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_messages.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_messages.json new file mode 100644 index 000000000000..eb16f1ae1e8a --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_messages.json @@ -0,0 +1,67 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "sent_by": { + "type": ["null", "string"] + }, + "sent_by_email": { + "type": ["null", "string"] + }, + "sent_from": { + "type": ["null", "string"] + }, + "sent_from_email": { + "type": ["null", "string"] + }, + "include_link_to_client_invoice": { + "type": ["null", "boolean"] + }, + "send_me_a_copy": { + "type": ["null", "boolean"] + }, + "thank_you": { + "type": ["null", "boolean"] + }, + "reminder": { + "type": ["null", "boolean"] + }, + "send_reminder_on": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date" + }, + "attach_pdf": { + "type": ["null", "boolean"] + }, + "event_type": { + "type": ["null", "string"] + }, + "recipients": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + } + }, + "subject": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_payments.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_payments.json new file mode 100644 index 000000000000..6b5890eb5aa1 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoice_payments.json @@ -0,0 +1,51 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "integer"] + }, + "paid_at": { + "type": ["null", "string"], + "format": "date" + }, + "paid_date": { + "type": ["null", "string"], + "format": "date" + }, + "recorded_by": { + "type": ["null", "string"] + }, + "recorded_by_email": { + "type": ["null", "string"] + }, + "notes": { + "type": ["null", "string"] + }, + "transaction_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date" + }, + "payment_gateway": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json new file mode 100644 index 000000000000..a9bc6037c4e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/invoices.json @@ -0,0 +1,141 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "client_key": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "string"] + }, + "purchase_order": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"] + }, + "due_amount": { + "type": ["null", "number"] + }, + "tax": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"] + }, + "tax2": { + "type": ["null", "integer"] + }, + "tax2_amount": { + "type": ["null", "number"] + }, + "discount": { + "type": ["null", "integer"] + }, + "discount_amount": { + "type": ["null", "integer"] + }, + "subject": { + "type": ["null", "string"] + }, + "notes": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "period_start": { + "type": ["null", "string"], + "format": "date" + }, + "period_end": { + "type": ["null", "string"], + "format": "date" + }, + "issue_date": { + "type": ["null", "string"], + "format": "date" + }, + "due_date": { + "type": ["null", "string"], + "format": "date" + }, + "payment_term": { + "type": ["null", "string"] + }, + "sent_at": { + "type": ["null", "string"], + "format": "date" + }, + "paid_at": { + "type": ["null", "string"] + }, + "paid_date": { + "type": ["null", "string"] + }, + "closed_at": { + "type": ["null", "string"] + }, + "recurring_invoice_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date" + }, + "currency": { + "type": ["null", "string"] + }, + "client": { + "type": ["null", "string"] + }, + "estimate": { + "type": ["null", "string"] + }, + "retainer": { + "type": ["null", "string"] + }, + "creator": { + "type": ["null", "string"] + }, + "line_items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "kind": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "unit_price": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "integer"] + }, + "taxed": { + "type": ["null", "boolean"] + }, + "taxed2": { + "type": ["null", "boolean"] + }, + "project": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_assignments.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_assignments.json new file mode 100644 index 000000000000..eaf7b2b642c4 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_assignments.json @@ -0,0 +1,60 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "is_project_manager": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "use_default_rates": { + "type": ["null", "boolean"] + }, + "budget": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + }, + "project": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + } + } + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "task_assignments": { + "type": ["null", "array"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_budget.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_budget.json new file mode 100644 index 000000000000..3742a85771d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/project_budget.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "project_id": { + "type": ["null", "integer"] + }, + "project_name": { + "type": ["null", "string"] + }, + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "budget_is_monthly": { + "type": ["null", "boolean"] + }, + "budget_by": { + "type": ["null", "string"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "budget": { + "type": ["null", "integer"] + }, + "budget_spent": { + "type": ["null", "integer"] + }, + "budget_remaining": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/projects.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/projects.json new file mode 100644 index 000000000000..639ea72f6527 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/projects.json @@ -0,0 +1,93 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "bill_by": { + "type": ["null", "string"] + }, + "budget": { + "type": ["null", "number"] + }, + "budget_by": { + "type": ["null", "string"] + }, + "budget_is_monthly": { + "type": ["null", "boolean"] + }, + "notify_when_over_budget": { + "type": ["null", "boolean"] + }, + "over_budget_notification_percentage": { + "type": ["null", "number"] + }, + "over_budget_notification_date": { + "type": ["null", "string"], + "format": "date" + }, + "show_budget_to_all": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "starts_on": { + "type": ["null", "string"], + "format": "date" + }, + "ends_on": { + "type": ["null", "string"] + }, + "is_billable": { + "type": ["null", "boolean"] + }, + "is_fixed_fee": { + "type": ["null", "boolean"] + }, + "notes": { + "type": ["null", "string"] + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + } + } + }, + "cost_budget": { + "type": ["null", "string"] + }, + "cost_budget_include_expenses": { + "type": ["null", "boolean"] + }, + "hourly_rate": { + "type": ["null", "number"] + }, + "fee": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/roles.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/roles.json new file mode 100644 index 000000000000..fce9c76fcf81 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/roles.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "user_ids": { + "type": ["null", "array"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/task_assignments.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/task_assignments.json new file mode 100644 index 000000000000..3180d55362bc --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/task_assignments.json @@ -0,0 +1,54 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "billable": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + }, + "budget": { + "type": ["null", "string"] + }, + "project": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + } + } + }, + "task": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/tasks.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/tasks.json new file mode 100644 index 000000000000..465336d3ce25 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/tasks.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "billable_by_default": { + "type": ["null", "boolean"] + }, + "default_hourly_rate": { + "type": ["null", "number"] + }, + "is_default": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_clients.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_clients.json new file mode 100644 index 000000000000..43a92dba12ab --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_clients.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "total_hours": { + "type": ["null", "number"] + }, + "billable_hours": { + "type": ["null", "number"] + }, + "currency": { + "type": ["null", "string"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_entries.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_entries.json new file mode 100644 index 000000000000..95acba6cffcd --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_entries.json @@ -0,0 +1,174 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "spent_date": { + "type": ["null", "string"], + "format": "date" + }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "client": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "project": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "task": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "user_assignment": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "is_project_manager": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "budget": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + } + } + }, + "task_assignment": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "billable": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + }, + "budget": { + "type": ["null", "string"] + } + } + }, + "hours": { + "type": ["null", "number"] + }, + "rounded_hours": { + "type": ["null", "number"] + }, + "notes": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "is_locked": { + "type": ["null", "boolean"] + }, + "locked_reason": { + "type": ["null", "string"] + }, + "is_closed": { + "type": ["null", "boolean"] + }, + "is_billed": { + "type": ["null", "boolean"] + }, + "timer_started_at": { + "type": ["null", "string"] + }, + "started_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "ended_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "is_running": { + "type": ["null", "boolean"] + }, + "invoice": { + "type": ["null", "string"] + }, + "external_reference": { + "type": ["null", "string"] + }, + "billable": { + "type": ["null", "boolean"] + }, + "budgeted": { + "type": ["null", "boolean"] + }, + "billable_rate": { + "type": ["null", "number"] + }, + "cost_rate": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_projects.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_projects.json new file mode 100644 index 000000000000..7577b038da40 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_projects.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "project_id": { + "type": ["null", "integer"] + }, + "project_name": { + "type": ["null", "string"] + }, + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "total_hours": { + "type": ["null", "integer"] + }, + "billable_hours": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_tasks.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_tasks.json new file mode 100644 index 000000000000..6c337cbc3749 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_tasks.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "task_id": { + "type": ["null", "integer"] + }, + "task_name": { + "type": ["null", "string"] + }, + "total_hours": { + "type": ["null", "integer"] + }, + "billable_hours": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_team.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_team.json new file mode 100644 index 000000000000..4971b00b087d --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/time_team.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "user_id": { + "type": ["null", "integer"] + }, + "user_name": { + "type": ["null", "string"] + }, + "is_contractor": { + "type": ["null", "boolean"] + }, + "total_hours": { + "type": ["null", "number"] + }, + "billable_hours": { + "type": ["null", "number"] + }, + "currency": { + "type": ["null", "string"] + }, + "billable_amount": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/uninvoiced.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/uninvoiced.json new file mode 100644 index 000000000000..b124db596ea9 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/uninvoiced.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "client_id": { + "type": ["null", "integer"] + }, + "client_name": { + "type": ["null", "string"] + }, + "project_id": { + "type": ["null", "integer"] + }, + "project_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "total_hours": { + "type": ["null", "integer"] + }, + "uninvoiced_hours": { + "type": ["null", "integer"] + }, + "uninvoiced_expenses": { + "type": ["null", "integer"] + }, + "uninvoiced_amount": { + "type": ["null", "integer"] + }, + "from": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/user_assignments.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/user_assignments.json new file mode 100644 index 000000000000..c5c87e75a231 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/user_assignments.json @@ -0,0 +1,57 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "is_project_manager": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "use_default_rates": { + "type": ["null", "boolean"] + }, + "budget": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "hourly_rate": { + "type": ["null", "number"] + }, + "project": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + } + } + }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/users.json b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/users.json new file mode 100644 index 000000000000..0a1329308ac8 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/schemas/users.json @@ -0,0 +1,71 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "telephone": { + "type": ["null", "string"] + }, + "timezone": { + "type": ["null", "string"] + }, + "has_access_to_all_future_projects": { + "type": ["null", "boolean"] + }, + "is_contractor": { + "type": ["null", "boolean"] + }, + "is_admin": { + "type": ["null", "boolean"] + }, + "is_project_manager": { + "type": ["null", "boolean"] + }, + "can_see_rates": { + "type": ["null", "boolean"] + }, + "can_create_projects": { + "type": ["null", "boolean"] + }, + "can_create_invoices": { + "type": ["null", "boolean"] + }, + "is_active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "weekly_capacity": { + "type": ["null", "integer"] + }, + "default_hourly_rate": { + "type": ["null", "number"] + }, + "cost_rate": { + "type": ["null", "number"] + }, + "roles": { + "type": ["null", "array"] + }, + "avatar_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/source.py b/airbyte-integrations/connectors/source-harvest/source_harvest/source.py new file mode 100644 index 000000000000..f53eee3e790b --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/source.py @@ -0,0 +1,127 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from typing import Any, List, Mapping, Tuple + +import pendulum +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from source_harvest.streams import ( + BillableRates, + Clients, + Company, + Contacts, + CostRates, + EstimateItemCategories, + EstimateMessages, + Estimates, + ExpenseCategories, + Expenses, + ExpensesCategories, + ExpensesClients, + ExpensesProjects, + ExpensesTeam, + InvoiceItemCategories, + InvoiceMessages, + InvoicePayments, + Invoices, + ProjectAssignments, + ProjectBudget, + Projects, + Roles, + TaskAssignments, + Tasks, + TimeClients, + TimeEntries, + TimeProjects, + TimeTasks, + TimeTeam, + Uninvoiced, + UserAssignments, + Users, +) + +from .auth import HarvestTokenAuthenticator + + +class SourceHarvest(AbstractSource): + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + try: + auth = HarvestTokenAuthenticator(token=config["api_token"], account_id=config["account_id"]) + replication_start_date = pendulum.parse(config["replication_start_date"]) + users_gen = Users(authenticator=auth, replication_start_date=replication_start_date).read_records( + sync_mode=SyncMode.full_refresh + ) + next(users_gen) + return True, None + except Exception as error: + return False, f"Unable to connect to Harvest API with the provided credentials - {repr(error)}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + auth = HarvestTokenAuthenticator(token=config["api_token"], account_id=config["account_id"]) + replication_start_date = pendulum.parse(config["replication_start_date"]) + from_date = replication_start_date.date() + + streams = [ + Clients(authenticator=auth, replication_start_date=replication_start_date), + Contacts(authenticator=auth, replication_start_date=replication_start_date), + Company(authenticator=auth), + Invoices(authenticator=auth, replication_start_date=replication_start_date), + InvoiceMessages(authenticator=auth, replication_start_date=replication_start_date), + InvoicePayments(authenticator=auth, replication_start_date=replication_start_date), + InvoiceItemCategories(authenticator=auth, replication_start_date=replication_start_date), + Estimates(authenticator=auth, replication_start_date=replication_start_date), + EstimateMessages(authenticator=auth, replication_start_date=replication_start_date), + EstimateItemCategories(authenticator=auth, replication_start_date=replication_start_date), + Expenses(authenticator=auth, replication_start_date=replication_start_date), + ExpenseCategories(authenticator=auth, replication_start_date=replication_start_date), + Tasks(authenticator=auth, replication_start_date=replication_start_date), + TimeEntries(authenticator=auth, replication_start_date=replication_start_date), + UserAssignments(authenticator=auth, replication_start_date=replication_start_date), + TaskAssignments(authenticator=auth, replication_start_date=replication_start_date), + Projects(authenticator=auth, replication_start_date=replication_start_date), + Roles(authenticator=auth, replication_start_date=replication_start_date), + Users(authenticator=auth, replication_start_date=replication_start_date), + BillableRates(authenticator=auth), + CostRates(authenticator=auth), + ProjectAssignments(authenticator=auth, replication_start_date=replication_start_date), + ExpensesClients(authenticator=auth, from_date=from_date), + ExpensesProjects(authenticator=auth, from_date=from_date), + ExpensesCategories(authenticator=auth, from_date=from_date), + ExpensesTeam(authenticator=auth, from_date=from_date), + Uninvoiced(authenticator=auth, from_date=from_date), + TimeClients(authenticator=auth, from_date=from_date), + TimeProjects(authenticator=auth, from_date=from_date), + TimeTasks(authenticator=auth, from_date=from_date), + TimeTeam(authenticator=auth, from_date=from_date), + ProjectBudget(authenticator=auth, from_date=from_date), + ] + + return streams diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json new file mode 100644 index 000000000000..adda14af013b --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/spec.json @@ -0,0 +1,33 @@ +{ + "documentationUrl": "https://hub.docker.com/r/airbyte/source-harvest", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Harvest Spec", + "type": "object", + "required": ["api_key", "account_id", "replication_start_date"], + "additionalProperties": false, + "properties": { + "api_key": { + "title": "API Key", + "description": "Harvest API Key.", + "airbyte_secret": true, + "type": "string" + }, + "account_id": { + "title": "Account ID", + "description": "Harvest account ID. Required for all Harvest requests in pair with API Key", + "airbyte_secret": true, + "type": "string" + }, + "replication_start_date": { + "title": "Replication Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": ["2017-01-25T00:00:00Z"], + "type": "string" + } + } + }, + "supportsIncremental": true, + "supported_destination_sync_modes": ["append"] +} diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py new file mode 100644 index 000000000000..8e7690c937b2 --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py @@ -0,0 +1,457 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from abc import ABC, abstractmethod +from typing import Any, Iterable, Mapping, MutableMapping, Optional +from urllib.parse import parse_qsl, urlparse + +import pendulum +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.http import HttpStream + + +class HarvestStream(HttpStream, ABC): + url_base = "https://api.harvestapp.com/v2/" + per_page = 50 + primary_key = "id" + + @property + def data_field(self) -> str: + """ + :return: Default field name to get data from response + """ + return self.name + + def path(self, **kwargs) -> str: + return self.name + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + stream_data = response.json() + if stream_data.get("next_page"): + return { + "page": stream_data["next_page"], + } + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + params["per_page"] = self.per_page + if next_page_token: + params.update(**next_page_token) + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + :return an iterable containing each record in the response + """ + stream_data = response.json() + + # depending on stream type we may get either: + # * nested records iterable in response object; + # * not nested records iterable; + # * single object to yield. + if self.data_field: + stream_data = response.json().get(self.data_field, []) + + if isinstance(stream_data, list): + yield from stream_data + else: + yield stream_data + + +class IncrementalHarvestStream(HarvestStream, ABC): + cursor_field = "updated_at" + + def __init__(self, replication_start_date: pendulum.datetime = None, **kwargs): + super().__init__(**kwargs) + self._replication_start_date = replication_start_date + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + latest_benchmark = latest_record[self.cursor_field] + if current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} + return {self.cursor_field: latest_benchmark} + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + replication_start_date = stream_state.get(self.cursor_field) or self._replication_start_date + params.update({"updated_since": replication_start_date}) + return params + + +class HarvestSubStream(HarvestStream): + @property + @abstractmethod + def path_template(self) -> str: + """ + :return: sub stream path template + """ + + @property + @abstractmethod + def parent_stream(self) -> IncrementalHarvestStream: + """ + :return: parent stream class + """ + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + items = self.parent_stream(authenticator=self.authenticator) + for item in items.read_records(sync_mode=SyncMode.full_refresh): + yield {"parent_id": item["id"]} + + def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: + return self.path_template.format(parent_id=stream_slice["parent_id"]) + + +class Contacts(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/clients-api/clients/contacts/ + """ + + +class Clients(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/clients-api/clients/clients/ + """ + + +class Company(HarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/company-api/company/company/ + """ + + primary_key = None + data_field = None + + +class Invoices(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/invoices-api/invoices/invoices/ + """ + + +class InvoiceMessages(HarvestSubStream, IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/invoices-api/invoices/invoice-messages/ + """ + + parent_stream = Invoices + path_template = "invoices/{parent_id}/messages" + + +class InvoicePayments(HarvestSubStream, IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/invoices-api/invoices/invoice-payments/ + """ + + parent_stream = Invoices + path_template = "invoices/{parent_id}/payments" + + +class InvoiceItemCategories(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/invoices-api/invoices/invoice-item-categories/ + """ + + +class Estimates(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/estimates-api/estimates/estimates/ + """ + + +class EstimateMessages(HarvestSubStream, IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/estimates-api/estimates/estimate-messages/ + """ + + parent_stream = Estimates + path_template = "estimates/{parent_id}/messages" + + +class EstimateItemCategories(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/estimates-api/estimates/estimate-item-categories/ + """ + + +class Expenses(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/expenses-api/expenses/expenses/ + """ + + +class ExpenseCategories(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/expenses-api/expenses/expense-categories/ + """ + + +class Tasks(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/tasks-api/tasks/tasks/ + """ + + +class TimeEntries(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/timesheets-api/timesheets/time-entries/ + """ + + +class UserAssignments(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/projects-api/projects/user-assignments/ + """ + + +class TaskAssignments(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/projects-api/projects/task-assignments/ + """ + + +class Projects(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/projects-api/projects/projects/ + """ + + +class Roles(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/roles-api/roles/roles/ + """ + + +class Users(IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/users-api/users/users/ + """ + + +class BillableRates(HarvestSubStream): + """ + Docs: https://help.getharvest.com/api-v2/users-api/users/billable-rates/ + """ + + parent_stream = Users + path_template = "users/{parent_id}/billable_rates" + + +class CostRates(HarvestSubStream): + """ + Docs: https://help.getharvest.com/api-v2/users-api/users/cost-rates/ + """ + + parent_stream = Users + path_template = "users/{parent_id}/cost_rates" + + +class ProjectAssignments(HarvestSubStream, IncrementalHarvestStream): + """ + Docs: https://help.getharvest.com/api-v2/users-api/users/project-assignments/ + """ + + parent_stream = Users + path_template = "users/{parent_id}/project_assignments" + + +class ReportsBase(HarvestStream, ABC): + data_field = "results" + date_param_template = "%Y%m%d" + + @property + @abstractmethod + def report_path(self): + """ + :return: report path suffix + """ + + def __init__(self, from_date: pendulum.date = None, **kwargs): + super().__init__(**kwargs) + + current_date = pendulum.now().date() + self._from_date = from_date or current_date.subtract(years=1) + # `to` date greater than `from` date causes an exception on Harvest + if self._from_date > current_date: + self._to_date = from_date + else: + self._to_date = current_date + + def request_params(self, stream_state, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, **kwargs) + current_date = pendulum.now() + # `from` and `to` params are required for reports calls + # min `from` value is current_date - 1 year + params.update({"from": self._from_date.strftime("%Y%m%d"), "to": current_date.strftime("%Y%m%d")}) + return params + + def path(self, **kwargs) -> str: + return f"reports/{self.report_path}" + + +class IncrementalReportsBase(ReportsBase, ABC): + cursor_field = "to" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + parsed_url = urlparse(response.url) + params = dict(parse_qsl(parsed_url.query)) + + records = response.json().get(self.data_field, []) + for record in records: + record.update( + { + "from": params.get("from", self._from_date.strftime(self.date_param_template)), + "to": params.get("to", self._to_date.strftime(self.date_param_template)), + } + ) + yield record + + def request_params(self, stream_state: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + stream_state = stream_state or {} + params = super().request_params(stream_state, **kwargs) + + # subtract `from` date by 1 year to avoid Harvest exception + # `from` date may not be less than `to` - 1 year + if stream_state.get(self.cursor_field): + cursor_date = pendulum.parse(stream_state[self.cursor_field]).date() + dates_diff = cursor_date - self._from_date + if dates_diff.years > 0 or dates_diff.years == 0 and dates_diff.remaining_days > 0: + self._from_date = cursor_date.subtract(years=1) + + # `from` and `to` params are required for reports calls + # min `from` value is current_date - 1 year + params.update( + { + "from": self._from_date.strftime(self.date_param_template), + "to": stream_state.get(self.cursor_field, self._to_date.strftime(self.date_param_template)), + } + ) + return params + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + latest_benchmark = latest_record[self.cursor_field] + if current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} + return {self.cursor_field: latest_benchmark} + + +class ExpensesClients(IncrementalReportsBase): + """ + Docs: https://help.getharvest.com/api-v2/reports-api/reports/expense-reports/#clients-report + """ + + report_path = "expenses/clients" + + +class ExpensesProjects(IncrementalReportsBase): + """ + Docs: https://help.getharvest.com/api-v2/reports-api/reports/expense-reports/#projects-report + """ + + report_path = "expenses/projects" + + +class ExpensesCategories(IncrementalReportsBase): + """ + Docs: https://help.getharvest.com/api-v2/reports-api/reports/expense-reports/#expense-categories-report + """ + + report_path = "expenses/categories" + + +class ExpensesTeam(IncrementalReportsBase): + """ + Docs: https://help.getharvest.com/api-v2/reports-api/reports/expense-reports/#team-report + """ + + report_path = "expenses/team" + + +class Uninvoiced(ReportsBase): + """ + Docs: https://help.getharvest.com/api-v2/reports-api/reports/uninvoiced-report/ + + TODO: `from`/`to` pagination does not work for `uninvoiced` stream. Look like a bug on Harvest side. Check out later. + """ + + report_path = "uninvoiced" + + +class TimeClients(IncrementalReportsBase): + """ + Docs: https://help.getharvest.com/api-v2/reports-api/reports/time-reports/#clients-report + """ + + report_path = "time/clients" + + +class TimeProjects(IncrementalReportsBase): + """ + Docs: https://help.getharvest.com/api-v2/reports-api/reports/time-reports/#projects-report + """ + + report_path = "time/projects" + + +class TimeTasks(IncrementalReportsBase): + """ + Docs: https://help.getharvest.com/api-v2/reports-api/reports/time-reports/#tasks-report + """ + + report_path = "time/tasks" + + +class TimeTeam(IncrementalReportsBase): + """ + Docs: https://help.getharvest.com/api-v2/reports-api/reports/time-reports/ (Team Report) + """ + + report_path = "time/team" + + +class ProjectBudget(ReportsBase): + """ + Docs: https://help.getharvest.com/api-v2/reports-api/reports/time-reports/#team-report + """ + + report_path = "project_budget" diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py new file mode 100644 index 000000000000..b8a8150b507f --- /dev/null +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/unit_test.py @@ -0,0 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +def test_example_method(): + assert True diff --git a/airbyte-integrations/connectors/source-posthog/.dockerignore b/airbyte-integrations/connectors/source-posthog/.dockerignore new file mode 100644 index 000000000000..eb58d656b2c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/.dockerignore @@ -0,0 +1,5 @@ +* +!main.py +!source_posthog +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-posthog/CHANGELOG.md b/airbyte-integrations/connectors/source-posthog/CHANGELOG.md new file mode 100644 index 000000000000..45fcf432379b --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +## 0.1.0 +Source implementation. diff --git a/airbyte-integrations/connectors/source-posthog/Dockerfile b/airbyte-integrations/connectors/source-posthog/Dockerfile new file mode 100644 index 000000000000..68e40265ab0d --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_posthog ./source_posthog +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-posthog diff --git a/airbyte-integrations/connectors/source-posthog/README.md b/airbyte-integrations/connectors/source-posthog/README.md new file mode 100644 index 000000000000..b6b88c84c6d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/README.md @@ -0,0 +1,129 @@ +# PostHog Source + +This is the repository for the PostHog source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/posthog). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-posthog:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/posthog) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_posthog/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source posthog test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-posthog:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-posthog:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-posthog:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-posthog:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-posthog:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-posthog:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unittest run: +``` +./gradlew :airbyte-integrations:connectors:source-posthog:unitTest +``` +To run acceptance and custom integration tests run: +``` +./gradlew :airbyte-integrations:connectors:source-posthog:IntegrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request +1. Pat yourself on the back for being an awesome contributor +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master diff --git a/airbyte-integrations/connectors/source-posthog/acceptance-test-config.yml b/airbyte-integrations/connectors/source-posthog/acceptance-test-config.yml new file mode 100644 index 000000000000..e6a43b889d40 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/acceptance-test-config.yml @@ -0,0 +1,22 @@ +connector_image: airbyte/source-posthog:dev +tests: + spec: + - spec_path: "source_posthog/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh new file mode 100644 index 000000000000..03945e44e7d4 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-posthog/build.gradle b/airbyte-integrations/connectors/source-posthog/build.gradle new file mode 100644 index 000000000000..2b7de1b343a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_posthog' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) +} + diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/__init__.py b/airbyte-integrations/connectors/source-posthog/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py new file mode 100644 index 000000000000..d98ac8aa3a1c --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-posthog/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..b8627dc493dd --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/configured_catalog.json @@ -0,0 +1,139 @@ +{ + "streams": [ + { + "stream": { + "name": "annotations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + }, + { + "stream": { + "name": "cohorts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + }, + { + "stream": { + "name": "events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["timestamp"], + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + }, + { + "stream": { + "name": "feature_flags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + }, + { + "stream": { + "name": "insights", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + }, + { + "stream": { + "name": "insights_path", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + }, + { + "stream": { + "name": "insights_sessions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + }, + { + "stream": { + "name": "persons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + }, + { + "stream": { + "name": "trends", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": null, + "default_cursor_field": null, + "source_defined_primary_key": [["id"]], + "namespace": null + }, + "sync_mode": "full_refresh", + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-posthog/integration_tests/invalid_config.json new file mode 100644 index 000000000000..1b3435fb9a6d --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "value1", + "start_date": "2021-01-01-T00:00:00.000000Z" +} diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/state.json b/airbyte-integrations/connectors/source-posthog/integration_tests/state.json new file mode 100644 index 000000000000..5a07faa9e2fe --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/state.json @@ -0,0 +1,6 @@ +{ + "feature_flags": { "id": 100000000000000000000000000 }, + "events": { "timestamp": "2121-04-13T18:13:51.504000+00:00" }, + "persons": { "created_at": "2121-04-13T18:13:54.269000Z" }, + "annotations": { "updated_at": "2121-05-27T14:09:29.961933Z" } +} diff --git a/airbyte-integrations/connectors/source-posthog/main.py b/airbyte-integrations/connectors/source-posthog/main.py new file mode 100644 index 000000000000..0d11ed6022de --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_posthog import SourcePosthog + +if __name__ == "__main__": + source = SourcePosthog() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-posthog/requirements.txt b/airbyte-integrations/connectors/source-posthog/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-posthog/sample_files/state.json b/airbyte-integrations/connectors/source-posthog/sample_files/state.json new file mode 100644 index 000000000000..31d52e9a4d56 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/sample_files/state.json @@ -0,0 +1,6 @@ +{ + "feature_flags": { "id": 697 }, + "events": { "timestamp": "2021-05-31T06:58:11.633000+00:00" }, + "persons": { "created_at": "2021-04-13T18:13:54.269000Z" }, + "annotations": { "updated_at": "2021-05-27T14:09:29.961933Z" } +} diff --git a/airbyte-integrations/connectors/source-posthog/setup.py b/airbyte-integrations/connectors/source-posthog/setup.py new file mode 100644 index 000000000000..eadf4460f11b --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/setup.py @@ -0,0 +1,46 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_posthog", + description="Source implementation for Posthog.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json"]}, + extras_require={"tests": TEST_REQUIREMENTS}, +) diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/__init__.py b/airbyte-integrations/connectors/source-posthog/source_posthog/__init__.py new file mode 100644 index 000000000000..efdacd2cff39 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/__init__.py @@ -0,0 +1,3 @@ +from .source import SourcePosthog + +__all__ = ["SourcePosthog"] diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/annotations.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/annotations.json new file mode 100644 index 000000000000..9b38f7176c25 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/annotations.json @@ -0,0 +1,55 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "content": { + "type": "string" + }, + "date_marker": { + "type": "string", + "format": "date-time" + }, + "creation_type": { + "type": "string" + }, + "dashboard_item": { + "type": "string" + }, + "created_by": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "uuid": { + "type": "string" + }, + "distinct_id": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "deleted": { + "type": "boolean" + }, + "scope": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/cohorts.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/cohorts.json new file mode 100644 index 000000000000..1c3547b9b76a --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/cohorts.json @@ -0,0 +1,73 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "days": { + "type": "string" + }, + "action_id": { + "type": "string" + }, + "properties": { + "type": "array", + "items": { + "type": "object" + } + } + } + } + }, + "deleted": { + "type": "boolean" + }, + "is_calculating": { + "type": "boolean" + }, + "created_by": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "uuid": { + "type": "string" + }, + "distinct_id": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "last_calculation": { + "type": "string" + }, + "errors_calculating": { + "type": "integer" + }, + "count": { + "type": "integer" + }, + "is_static": { + "type": "boolean" + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events.json new file mode 100644 index 000000000000..6d9f0f1b1b21 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events.json @@ -0,0 +1,44 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "distinct_id": { + "type": "string" + }, + "event": { + "type": "string" + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "person": { + "type": "object", + "properties": { + "is_identified": { + "type": "boolean" + }, + "distinct_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "properties": { + "type": "object" + } + } + }, + "elements": { + "type": "array", + "items": { + "type": "string" + } + }, + "elements_chain": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events_sessions.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events_sessions.json new file mode 100644 index 000000000000..9940b2c221a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events_sessions.json @@ -0,0 +1,196 @@ +{ + "type": "object", + "properties": { + "distinct_id": { + "type": "string" + }, + "global_session_id": { + "type": "integer" + }, + "length": { + "type": "integer" + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "end_time": { + "type": "string", + "format": "date-time" + }, + "event_count": { + "type": "integer" + }, + "events": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "distinct_id": { + "type": "string" + }, + "properties": { + "type": "object", + "properties": { + "$os": { + "type": "string" + }, + "$browser": { + "type": "string" + }, + "$device_type": { + "type": "string" + }, + "$current_url": { + "type": "string" + }, + "$host": { + "type": "string" + }, + "$pathname": { + "type": "string" + }, + "$browser_version": { + "type": "integer" + }, + "$screen_height": { + "type": "integer" + }, + "$screen_width": { + "type": "integer" + }, + "$lib": { + "type": "string" + }, + "$lib_version": { + "type": "string" + }, + "$insert_id": { + "type": "string" + }, + "$time": { + "type": "number" + }, + "distinct_id": { + "type": "string" + }, + "$device_id": { + "type": "string" + }, + "$initial_referrer": { + "type": "string" + }, + "$initial_referring_domain": { + "type": "string" + }, + "$referrer": { + "type": "string" + }, + "$referring_domain": { + "type": "string" + }, + "$active_feature_flags": { + "type": "array", + "items": { + "type": "string" + } + }, + "$event_type": { + "type": "string" + }, + "$ce_version": { + "type": "integer" + }, + "token": { + "type": "string" + }, + "$ip": { + "type": "string" + } + } + }, + "event": { + "type": "string" + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "person": { + "type": "string" + }, + "elements": { + "type": "array", + "items": { + "type": "object", + "properties": { + "event": { + "type": "string" + }, + "text": { + "type": "string" + }, + "tag_name": { + "type": "string" + }, + "attr_class": { + "type": "string" + }, + "href": { + "type": "string" + }, + "attr_id": { + "type": "string" + }, + "nth_child": { + "type": "integer" + }, + "nth_of_type": { + "type": "integer" + }, + "attributes": { + "type": "object" + }, + "order": { + "type": "integer" + } + } + } + }, + "elements_chain": { + "type": "string" + } + } + } + }, + "properties": { + "type": "object" + }, + "matching_events": { + "type": "array", + "items": { + "type": "string" + } + }, + "email": { + "type": "string" + }, + "session_recordings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "viewed": { + "type": "boolean" + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/feature_flags.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/feature_flags.json new file mode 100644 index 000000000000..54a8ebefa0d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/feature_flags.json @@ -0,0 +1,55 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "key": { + "type": "string" + }, + "rollout_percentage": { + "type": "integer" + }, + "filters": { + "type": "object", + "properties": { + "properties": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "deleted": { + "type": "boolean" + }, + "active": { + "type": "boolean" + }, + "created_by": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "distinct_id": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + }, + "created_at": { + "type": "string", + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json new file mode 100644 index 000000000000..8e03af5b0e9a --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json @@ -0,0 +1,124 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "filters": { + "type": "object", + "properties": { + "events": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "math": { + "type": "string" + }, + "name": { + "type": "string" + }, + "type": { + "type": "string" + }, + "order": { + "type": "integer" + }, + "properties": { + "type": "array", + "items": { + "type": "string" + } + }, + "math_property": { + "type": "string" + } + } + } + }, + "display": { + "type": "string" + }, + "filters": { + "type": "array", + "items": { + "type": "string" + } + }, + "insight": { + "type": "string" + }, + "session": { + "type": "string" + }, + "interval": { + "type": "string" + }, + "pagination": { + "type": "object" + } + } + }, + "filters_hash": { + "type": "string" + }, + "order": { + "type": "string" + }, + "deleted": { + "type": "boolean" + }, + "dashboard": { + "type": "string" + }, + "layouts": { + "type": "object" + }, + "color": { + "type": "string" + }, + "last_refresh": { + "type": "string", + "format": "date-time" + }, + "refreshing": { + "type": "boolean" + }, + "result": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "saved": { + "type": "boolean" + }, + "created_by": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "uuid": { + "type": "string" + }, + "distinct_id": { + "type": "string" + }, + "first_name": { + "type": "string" + }, + "email": { + "type": "string" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_path.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_path.json new file mode 100644 index 000000000000..15677b9a26a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_path.json @@ -0,0 +1,20 @@ +{ + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "source_id": { + "type": "string" + }, + "target": { + "type": "string" + }, + "target_id": { + "type": "string" + }, + "value": { + "type": "integer" + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_sessions.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_sessions.json new file mode 100644 index 000000000000..c13199f9d299 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_sessions.json @@ -0,0 +1,32 @@ +{ + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "days": { + "type": "array", + "items": { + "type": "string" + } + }, + "label": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "chartLabel": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/persons.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/persons.json new file mode 100644 index 000000000000..5c1f5120b9ee --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/persons.json @@ -0,0 +1,24 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "distinct_ids": { + "type": "array", + "items": { + "type": "string" + } + }, + "properties": { + "type": "object" + }, + "created_at": { + "type": "string", + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/sessions.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/sessions.json new file mode 100644 index 000000000000..b9e6251107b6 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/sessions.json @@ -0,0 +1,158 @@ +{ + "type": "object", + "properties": { + "distinct_id": { + "type": "string" + }, + "global_session_id": { + "type": "integer" + }, + "length": { + "type": "integer" + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "end_time": { + "type": "string", + "format": "date-time" + }, + "event_count": { + "type": "integer" + }, + "events": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "distinct_id": { + "type": "string" + }, + "properties": { + "type": "object", + "properties": { + "$os": { + "type": "string" + }, + "$browser": { + "type": "string" + }, + "$device_type": { + "type": "string" + }, + "$current_url": { + "type": "string" + }, + "$host": { + "type": "string" + }, + "$pathname": { + "type": "string" + }, + "$browser_version": { + "type": "integer" + }, + "$screen_height": { + "type": "integer" + }, + "$screen_width": { + "type": "integer" + }, + "$lib": { + "type": "string" + }, + "$lib_version": { + "type": "string" + }, + "$insert_id": { + "type": "string" + }, + "$time": { + "type": "number" + }, + "distinct_id": { + "type": "string" + }, + "$device_id": { + "type": "string" + }, + "$initial_referrer": { + "type": "string" + }, + "$initial_referring_domain": { + "type": "string" + }, + "$referrer": { + "type": "string" + }, + "$referring_domain": { + "type": "string" + }, + "$active_feature_flags": { + "type": "array", + "items": { + "type": "string" + } + }, + "token": { + "type": "string" + }, + "$ip": { + "type": "string" + } + } + }, + "event": { + "type": "string" + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "person": { + "type": "string" + }, + "elements": { + "type": "array", + "items": { + "type": "string" + } + }, + "elements_chain": { + "type": "string" + } + } + } + }, + "properties": { + "type": "object" + }, + "matching_events": { + "type": "array", + "items": { + "type": "string" + } + }, + "email": { + "type": "string" + }, + "session_recordings": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "viewed": { + "type": "boolean" + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/trends.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/trends.json new file mode 100644 index 000000000000..381934a6f624 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/trends.json @@ -0,0 +1,58 @@ +{ + "type": "object", + "properties": { + "action": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + }, + "order": { + "type": "string" + }, + "name": { + "type": "string" + }, + "math": { + "type": "string" + }, + "math_property": { + "type": "string" + }, + "properties": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "label": { + "type": "string" + }, + "count": { + "type": "number" + }, + "data": { + "type": "array", + "items": { + "type": "number" + } + }, + "labels": { + "type": "array", + "items": { + "type": "string" + } + }, + "days": { + "type": "array", + "items": { + "type": "string" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/source.py b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py new file mode 100644 index 000000000000..17b7ce566144 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py @@ -0,0 +1,81 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from typing import Any, List, Mapping, Tuple + +import pendulum +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +from .streams import ( # EventsSessions, + Annotations, + Cohorts, + Events, + FeatureFlags, + Insights, + InsightsPath, + InsightsSessions, + Persons, + Trends, +) + + +class SourcePosthog(AbstractSource): + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + try: + _ = pendulum.parse(config["start_date"], strict=True) + authenticator = TokenAuthenticator(token=config["api_key"]) + stream = Cohorts(authenticator=authenticator) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + _ = next(records) + return True, None + except Exception as e: + return False, repr(e) + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + event/sessions stream is dynamic. Probably, it contains a list of CURRENT sessions. + In Next day session may expire and wont be available via this endpoint. + So we need a dynamic load data before tests. + This stream was requested to be removed due to this reason. + """ + authenticator = TokenAuthenticator(token=config["api_key"]) + return [ + Annotations(authenticator=authenticator, start_date=config["start_date"]), + Cohorts(authenticator=authenticator), + Events(authenticator=authenticator, start_date=config["start_date"]), + # disabled because the endpoint returns only active sessions and they have TTL=24h + # so most of the time it will be empty + # EventsSessions(authenticator=authenticator), + FeatureFlags(authenticator=authenticator), + Insights(authenticator=authenticator), + InsightsPath(authenticator=authenticator), + InsightsSessions(authenticator=authenticator), + Persons(authenticator=authenticator), + Trends(authenticator=authenticator), + ] diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json b/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json new file mode 100644 index 000000000000..59e6afb5cb1f --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json @@ -0,0 +1,24 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/posthog", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "PostHog Spec", + "type": "object", + "required": ["api_key", "start_date"], + "additionalProperties": false, + "properties": { + "start_date": { + "title": "Start Date", + "type": "string", + "description": "The date from which you'd like to replicate the data", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": "2021-01-01T00:00:00.000000Z" + }, + "api_key": { + "type": "string", + "airbyte_secret": true, + "description": "API Key. See the docs for information on how to generate this key." + } + } + } +} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py b/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py new file mode 100644 index 000000000000..3f9fce5a9ff4 --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py @@ -0,0 +1,230 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import math +import urllib.parse +from abc import ABC, abstractmethod +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import requests +from airbyte_cdk.sources.streams.http import HttpStream + + +class PosthogStream(HttpStream, ABC): + url_base = "https://app.posthog.com/api/" + primary_key = "id" + data_field = "results" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + resp_json = response.json() + if resp_json.get("next"): + next_query_string = urllib.parse.urlsplit(resp_json["next"]).query + params = dict(urllib.parse.parse_qsl(next_query_string)) + return params + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + return {"Content-Type": "application/json", "User-Agent": "posthog-python/1.4.0"} + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + response_json = response.json() + yield from response_json.get(self.data_field, []) + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + params = {} + if next_page_token: + params.update(next_page_token) + return params + + +class IncrementalPosthogStream(PosthogStream, ABC): + """ + Because endpoints has descending order we need to save initial state value to know when to stop pagination. + start_date is used to as a min date to filter on. + """ + + state_checkpoint_interval = math.inf + + def __init__(self, start_date: str, **kwargs): + super().__init__(**kwargs) + self._start_date = start_date + self._initial_state = None # we need to keep it here because next_page_token doesn't accept state argument + + @property + @abstractmethod + def cursor_field(self) -> str: + """ + Defining a cursor field indicates that a stream is incremental, so any incremental stream must extend this class + and define a cursor field. + """ + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + Return next page token until we reach the page with records older than state/start_date + """ + min_state = self._initial_state or self._start_date + response_json = response.json() + data = response_json.get(self.data_field, []) + latest_record = data[-1] if data else None # records are ordered so we check only last one + + if not latest_record or latest_record[self.cursor_field] > min_state: + return super().next_page_token(response=response) + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + latest_state = latest_record.get(self.cursor_field) + current_state = current_stream_state.get(self.cursor_field) or latest_state + return {self.cursor_field: max(latest_state, current_state)} + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + data = super().parse_response(response=response, stream_state=stream_state, **kwargs) + for record in data: + if record.get(self.cursor_field) >= stream_state.get(self.cursor_field, self._start_date): + yield record + + +class Annotations(IncrementalPosthogStream): + """ + Docs: https://posthog.com/docs/api/annotations + """ + + cursor_field = "updated_at" + + def path(self, **kwargs) -> str: + return "annotation" + + def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, **kwargs) + params["order"] = f"-{self.cursor_field}" # sort descending + return params + + +class Cohorts(PosthogStream): + """ + Docs: https://posthog.com/docs/api/cohorts + normal ASC sorting. But without filters like `since` + """ + + def path(self, **kwargs) -> str: + return "cohort" + + +class Events(IncrementalPosthogStream): + """ + Docs: https://posthog.com/docs/api/events + """ + + cursor_field = "timestamp" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "event" + + def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, **kwargs) + since_value = stream_state.get(self.cursor_field) or self._start_date + since_value = max(since_value, self._start_date) + params["after"] = since_value + return params + + +class EventsSessions(PosthogStream): + """ + Docs: https://posthog.com/docs/api/events + """ + + data_field = "result" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "event/sessions" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + resp_json = response.json() + return resp_json.get("pagination") + + +class FeatureFlags(PosthogStream): + """ + Docs: https://posthog.com/docs/api/feature-flags + """ + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "feature_flag" + + +class Insights(PosthogStream): + """ + Docs: https://posthog.com/docs/api/insights + Endpoint does not support incremental read because id, created_at and last_refresh are ordered in any particular way + """ + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "insight" + + +class InsightsPath(PosthogStream): + """ + Docs: https://posthog.com/docs/api/insights + """ + + data_field = "result" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "insight/path" + + +class InsightsSessions(PosthogStream): + """ + Docs: https://posthog.com/docs/api/insights + """ + + data_field = "result" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "insight/session" + + +class Persons(PosthogStream): + """ + Docs: https://posthog.com/docs/api/people + """ + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "person" + + +class Trends(PosthogStream): + """ + Docs: https://posthog.com/docs/api/insights + """ + + data_field = "result" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "insight/trend" diff --git a/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py new file mode 100644 index 000000000000..ca38996aeecf --- /dev/null +++ b/airbyte-integrations/connectors/source-posthog/unit_tests/unit_test.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from airbyte_cdk.logger import AirbyteLogger +from source_posthog import SourcePosthog + + +def test_client_wrong_credentials(): + source = SourcePosthog() + status, error = source.check_connection(logger=AirbyteLogger(), config={"api_key": "blahblah"}) + assert not status diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/Dockerfile b/airbyte-integrations/connectors/source-scaffold-java-jdbc/Dockerfile new file mode 100644 index 000000000000..76937a24e3cc --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/Dockerfile @@ -0,0 +1,13 @@ +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte + +ENV APPLICATION source-scaffold-java-jdbc + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 + +# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-scaffold-java-jdbc \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle b/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle new file mode 100644 index 000000000000..724ee928de6f --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle @@ -0,0 +1,28 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.source.scaffold-java-jdbc.ScaffoldJavaJdbcSource' +} + +dependencies { + implementation project(':airbyte-db') + implementation project(':airbyte-integrations:bases:base-java') + implementation project(':airbyte-protocol:models') + implementation project(':airbyte-integrations:connectors:source-jdbc') + + //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' + + testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) + + testImplementation 'org.apache.commons:commons-lang3:3.11' + + integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-scaffold-java-jdbc') + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') + + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java new file mode 100644 index 000000000000..a5a98384194e --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java @@ -0,0 +1,70 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.scaffold_java_jdbc; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.db.jdbc.NoOpJdbcStreamingQueryConfiguration; +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ScaffoldJavaJdbcSource extends AbstractJdbcSource implements Source { + + private static final Logger LOGGER = LoggerFactory.getLogger(ScaffoldJavaJdbcSource.class); + + // TODO insert your driver name. Ex: "com.microsoft.sqlserver.jdbc.SQLServerDriver" + static final String DRIVER_CLASS = "driver_name_here"; + + public ScaffoldJavaJdbcSource() { + // By default NoOpJdbcStreamingQueryConfiguration class is used, but may be updated. See see example + // MssqlJdbcStreamingQueryConfiguration + super(DRIVER_CLASS, new NoOpJdbcStreamingQueryConfiguration()); + } + + // TODO The config is based on spec.json, update according to your DB + @Override + public JsonNode toJdbcConfig(JsonNode aqqConfig) { + // TODO create DB config. Ex: "Jsons.jsonNode(ImmutableMap.builder().put("username", + // userName).put("password", pas)...build()); + return null; + } + + @Override + public Set getExcludedInternalSchemas() { + // TODO Add tables to exaclude, Ex "INFORMATION_SCHEMA", "sys", "spt_fallback_db", etc + return Set.of(""); + } + + public static void main(String[] args) throws Exception { + final Source source = new ScaffoldJavaJdbcSource(); + LOGGER.info("starting source: {}", ScaffoldJavaJdbcSource.class); + new IntegrationRunner(source).run(args); + LOGGER.info("completed source: {}", ScaffoldJavaJdbcSource.class); + } + +} diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json new file mode 100644 index 000000000000..42f60d956bbd --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/resources/spec.json @@ -0,0 +1,55 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/source/mysql", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "ScaffoldJavaJdbc Source Spec", + "type": "object", + "required": ["host", "port", "database", "username", "replication_method"], + "additionalProperties": false, + "properties": { + "host": { + "description": "Hostname of the database.", + "type": "string", + "order": 0 + }, + "port": { + "description": "Port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 3306, + "examples": ["3306"], + "order": 1 + }, + "database": { + "description": "Name of the database.", + "type": "string", + "order": 2 + }, + "username": { + "description": "Username to use to access the database.", + "type": "string", + "order": 3 + }, + "password": { + "description": "Password associated with the username.", + "type": "string", + "airbyte_secret": true, + "order": 4 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the jdbc url string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)", + "type": "string", + "order": 5 + }, + "replication_method": { + "type": "string", + "title": "Replication Method", + "description": "Replication method to use for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "order": 6, + "default": "STANDARD", + "enum": ["STANDARD", "CDC"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java new file mode 100644 index 000000000000..6c330ef86c2b --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java @@ -0,0 +1,87 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.scaffold_java_jdbc; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConnectorSpecification; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +public class ScaffoldJavaJdbcSourceAcceptanceTest extends SourceAcceptanceTest { + + private JsonNode config; + + @Override + protected void setup(TestDestinationEnv testEnv) throws Exception { + // TODO create new container. Ex: "new OracleContainer("epiclabs/docker-oracle-xe-11g");" + // TODO make container started. Ex: "container.start();" + // TODO init JsonNode config + // TODO crete airbyte Database object "Databases.createJdbcDatabase(...)" + // TODO insert test data to DB. Ex: "database.execute(connection-> ...)" + // TODO close Database. Ex: "database.close();" + } + + @Override + protected void tearDown(TestDestinationEnv testEnv) { + // TODO close container that was initialized in setup() method. Ex: "container.close();" + } + + @Override + protected String getImageName() { + return "airbyte/source-scaffold-java-jdbc:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + // TODO Return the ConfiguredAirbyteCatalog with ConfiguredAirbyteStream objects + return null; + } + + @Override + protected List getRegexTests() { + return Collections.emptyList(); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } + +} diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcJdbcSourceAcceptanceTest.java new file mode 100644 index 000000000000..7fcd2fbdfcf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcJdbcSourceAcceptanceTest.java @@ -0,0 +1,88 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.scaffold_java_jdbc; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +class ScaffoldJavaJdbcJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(ScaffoldJavaJdbcJdbcSourceAcceptanceTest.class); + + // TODO declare a test container for DB. EX: org.testcontainers.containers.OracleContainer + + @BeforeAll + static void init() { + // Oracle returns uppercase values + // TODO init test container. Ex: "new OracleContainer("epiclabs/docker-oracle-xe-11g")" + // TODO start container. Ex: "container.start();" + } + + @BeforeEach + public void setup() throws Exception { + // TODO init config. Ex: "config = Jsons.jsonNode(ImmutableMap.builder().put("host", + // host).put("port", port)....build()); + super.setup(); + } + + @AfterEach + public void tearDown() { + // TODO clean used resources + } + + @Override + public AbstractJdbcSource getSource() { + return new ScaffoldJavaJdbcSource(); + } + + @Override + public boolean supportsSchemas() { + // TODO check if your db supports it and update method accordingly + return false; + } + + @Override + public JsonNode getConfig() { + return config; + } + + @Override + public String getDriverClass() { + return ScaffoldJavaJdbcSource.DRIVER_CLASS; + } + + @AfterAll + static void cleanUp() { + // TODO close the container. Ex: "container.close();" + } + +} diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceTests.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceTests.java new file mode 100644 index 000000000000..0509ee532110 --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceTests.java @@ -0,0 +1,51 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.source.scaffold_java_jdbc; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.db.Database; +import org.junit.jupiter.api.Test; + +public class ScaffoldJavaJdbcSourceTests { + + private JsonNode config; + private Database database; + + @Test + public void testSettingTimezones() throws Exception { + // TODO init your container. Ex: "new + // org.testcontainers.containers.MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense();" + // TODO start the container. Ex: "container.start();" + // TODO prepare DB config. Ex: "config = getConfig(container, dbName, + // "serverTimezone=Europe/London");" + // TODO create DB, grant all privileges, etc. + // TODO check connection status. Ex: "AirbyteConnectionStatus check = new + // ScaffoldJavaJdbcGenericSource().check(config);" + // TODO assert connection status. Ex: "assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, + // check.getStatus());" + // TODO cleanup used resources and close used container. Ex: "container.close();" + } + +} diff --git a/airbyte-integrations/connectors/source-shopify/.dockerignore b/airbyte-integrations/connectors/source-shopify/.dockerignore new file mode 100644 index 000000000000..d4dc46625588 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_shopify +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-shopify/CHANGELOG.md b/airbyte-integrations/connectors/source-shopify/CHANGELOG.md new file mode 100644 index 000000000000..f5ade60dd22e --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/CHANGELOG.md @@ -0,0 +1,7 @@ +# Shopify Source CHANGELOG + +| Date | Released Version | Notes | +| :--- | :--- | :--- | +| `2021-06-08` | `0.1.4` | `Added Order Risks Stream` | +| `2021-06-07` | `0.1.3` | `Initial Python CDK Airbyte Source Shopify Release` | + diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile new file mode 100644 index 000000000000..87012f7d2e7e --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_shopify ./source_shopify +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.name=airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/README.md b/airbyte-integrations/connectors/source-shopify/README.md new file mode 100644 index 000000000000..d1ad9e2254ba --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/README.md @@ -0,0 +1,139 @@ +# Shopify Source + +This is the repository for the Shopify CDK source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/shopify). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python3 -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Once you finished with installing python requirements: +``` +deactivate +``` + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-shopify:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/shopify) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_shopify/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source shopify test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +Use your .venv inside your connector in order to proceed: +``` +python3 main.py spec +python3 main.py check --config secrets/config.json +python3 main.py discover --config secrets/config.json +python3 main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-shopify:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-shopify:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-shopify:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shopify:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shopify:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-shopify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python3 -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python3 -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python3 -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-shopify:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-shopify:integrationTest +``` +To build final build the connector: +``` +./gradlew :airbyte-integrations:connectors:source-shopify:build +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml new file mode 100644 index 000000000000..225937171f2c --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml @@ -0,0 +1,25 @@ +connector_image: airbyte/source-shopify:dev +tests: + spec: + - spec_path: "source_shopify/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + # When account doesn't have the Refunds + configured_catalog_path: "integration_tests/no_refunds_catalog.json" + validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + cursor_paths: + charges: [ "id" ] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-shopify/acceptance-test-docker.sh new file mode 100644 index 000000000000..1425ff74f151 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-shopify/build.gradle b/airbyte-integrations/connectors/source-shopify/build.gradle new file mode 100644 index 000000000000..009dd424b34e --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_shopify' +} diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/__init__.py b/airbyte-integrations/connectors/source-shopify/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..3d51ab878498 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json @@ -0,0 +1,32 @@ +{ + "orders": { + "id": 2497723957406 + }, + "collects": { + "id": 17279451103367 + }, + "products": { + "id": 5059311534215 + }, + "customers": { + "id": 3664773775518 + }, + "metafields": { + "id": 11269168529474 + }, + "transactions": { + "id": 3125579481246 + }, + "order_refunds": { + "id": 639978373255 + }, + "custom_collections": { + "id": 153276284994 + }, + "abandoned_checkouts": { + "id": 14254866366622 + }, + "order_risks": { + "id": 5933004390558 + } +} diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py new file mode 100644 index 000000000000..eeb4a2d3e02e --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..6e4c7194eac2 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json @@ -0,0 +1,3406 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": { + "type": ["null", "object"], + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "orders", + "json_schema": { + "properties": { + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "app_id": { + "type": ["null", "integer"] + }, + "browser_ip": { + "type": ["null", "string"] + }, + "buyer_accepts_marketing": { + "type": ["null", "boolean"] + }, + "cancel_reason": { + "type": ["null", "string"] + }, + "cancelled_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "cart_token": { + "type": ["null", "string"] + }, + "checkout_id": { + "type": ["null", "integer"] + }, + "checkout_token": { + "type": ["null", "string"] + }, + "client_details": { + "type": ["null", "object"], + "properties": { + "accept_language": { + "type": ["null", "string"] + }, + "browser_height": { + "type": ["null", "integer"] + }, + "browser_ip": { + "type": ["null", "string"] + }, + "browser_width": { + "type": ["null", "integer"] + }, + "session_hash": { + "type": ["null", "string"] + }, + "user_agent": { + "type": ["null", "string"] + } + } + }, + "closed_at": { + "type": ["null", "string"] + }, + "confirmed": { + "type": ["null", "boolean"] + }, + "contact_email": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "current_subtotal_price": { + "type": ["null", "string"] + }, + "current_subtotal_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "current_total_discounts": { + "type": ["null", "string"] + }, + "current_total_discounts_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "current_total_duties_set": { + "type": ["null", "string"] + }, + "current_total_price": { + "type": ["null", "string"] + }, + "current_total_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "current_total_tax": { + "type": ["null", "string"] + }, + "current_total_tax_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "customer_locale": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "string"] + }, + "discount_codes": { + "type": ["null", "array"] + }, + "email": { + "type": ["null", "string"] + }, + "financial_status": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gateway": { + "type": ["null", "string"] + }, + "landing_site": { + "type": ["null", "string"] + }, + "landing_site_ref": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "note": { + "type": ["null", "string"] + }, + "note_attributes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "number": { + "type": ["null", "integer"] + }, + "order_number": { + "type": ["null", "integer"] + }, + "order_status_url": { + "type": ["null", "string"] + }, + "original_total_duties_set": { + "type": ["null", "string"] + }, + "payment_gateway_names": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "phone": { + "type": ["null", "string"] + }, + "presentment_currency": { + "type": ["null", "string"] + }, + "processed_at": { + "type": ["null", "string"] + }, + "processing_method": { + "type": ["null", "string"] + }, + "reference": { + "type": ["null", "string"] + }, + "referring_site": { + "type": ["null", "string"] + }, + "source_identifier": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "source_url": { + "type": ["null", "string"] + }, + "subtotal_price": { + "type": ["null", "string"] + }, + "subtotal_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "tags": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "price": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + } + } + } + }, + "taxes_included": { + "type": ["null", "boolean"] + }, + "test": { + "type": ["null", "boolean"] + }, + "token": { + "type": ["null", "string"] + }, + "total_discounts": { + "type": ["null", "string"] + }, + "total_discounts_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "total_line_items_price": { + "type": ["null", "string"] + }, + "total_line_items_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "total_outstanding": { + "type": ["null", "string"] + }, + "total_price": { + "type": ["null", "string"] + }, + "total_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "total_price_usd": { + "type": ["null", "string"] + }, + "total_shipping_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "total_tax": { + "type": ["null", "string"] + }, + "total_tax_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "total_tip_received": { + "type": ["null", "string"] + }, + "total_weight": { + "type": ["null", "integer"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "string"] + }, + "billing_address": { + "type": ["null", "object"], + "properties": { + "first_name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "note": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "phone": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "accepts_marketing_updated_at": { + "type": ["null", "string"] + }, + "marketing_opt_in_level": { + "type": ["null", "string"] + }, + "tax_exemptions": { + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "country_name": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "boolean"] + } + } + } + } + }, + "discount_applications": { + "type": ["null", "array"] + }, + "fulfillments": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "receipt": { + "type": ["null", "object"] + }, + "service": { + "type": ["null", "string"] + }, + "shipment_status": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "tracking_company": { + "type": ["null", "string"] + }, + "tracking_number": { + "type": ["null", "string"] + }, + "tracking_numbers": { + "type": ["null", "array"] + }, + "tracking_url": { + "type": ["null", "string"] + }, + "tracking_urls": { + "type": ["null", "array"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "line_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "destination_location": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + } + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "grams": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "origin_location": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + } + }, + "price": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "properties": { + "type": ["null", "array"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "sku": { + "type": ["null", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + }, + "total_discount": { + "type": ["null", "string"] + }, + "total_discount_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "variant_id": { + "type": ["null", "integer"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "price": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "rate": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + } + } + } + }, + "duties": { + "type": ["null", "array"] + }, + "discount_allocations": { + "type": ["null", "array"] + } + } + } + } + } + } + }, + "line_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "destination_location": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + } + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "grams": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "origin_location": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + } + }, + "price": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "properties": { + "type": ["null", "array"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "sku": { + "type": ["null", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + }, + "total_discount": { + "type": ["null", "string"] + }, + "total_discount_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "variant_id": { + "type": ["null", "integer"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "price": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "rate": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + } + } + } + }, + "duties": { + "type": ["null", "array"] + }, + "discount_allocations": { + "type": ["null", "array"] + } + } + } + }, + "payment_details": { + "type": ["null", "object"], + "properties": { + "credit_card_bin": { + "type": ["null", "string"] + }, + "avs_result_code": { + "type": ["null", "string"] + }, + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + } + } + }, + "refunds": { + "type": ["null", "array"] + }, + "shipping_address": { + "type": ["null", "object"], + "properties": { + "first_name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + } + } + }, + "shipping_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "carrier_identifier": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "delivery_category": { + "type": ["null", "string"] + }, + "discounted_price": { + "type": ["null", "string"] + }, + "discounted_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "phone": { + "type": ["null", "string"] + }, + "price": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "requested_fulfillment_service_id": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"] + }, + "discount_allocations": { + "type": ["null", "array"] + } + } + } + } + } + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "products", + "json_schema": { + "properties": { + "published_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "published_scope": { + "type": ["null", "string"] + }, + "vendor": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "body_html": { + "type": ["null", "string"] + }, + "product_type": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "options": { + "type": ["null", "array"], + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "values": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "id": { + "type": ["null", "integer"] + }, + "position": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + } + }, + "image": { + "properties": { + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "variant_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "height": { + "type": ["null", "integer"] + }, + "alt": { + "type": ["null", "string"] + }, + "src": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "handle": { + "type": ["null", "string"] + }, + "images": { + "type": ["null", "array"], + "items": { + "properties": { + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "variant_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "height": { + "type": ["null", "integer"] + }, + "alt": { + "type": ["null", "string"] + }, + "src": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + } + }, + "template_suffix": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "variants": { + "type": ["null", "array"], + "items": { + "properties": { + "barcode": { + "type": ["null", "string"] + }, + "tax_code": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "weight_unit": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "position": { + "type": ["null", "integer"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "image_id": { + "type": ["null", "integer"] + }, + "inventory_policy": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "inventory_item_id": { + "type": ["null", "integer"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "weight": { + "type": ["null", "number"] + }, + "inventory_management": { + "type": ["null", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "option1": { + "type": ["null", "string"] + }, + "compare_at_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "option2": { + "type": ["null", "string"] + }, + "old_inventory_quantity": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "inventory_quantity": { + "type": ["null", "integer"] + }, + "grams": { + "type": ["null", "integer"] + }, + "option3": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "abandoned_checkouts", + "json_schema": { + "type": "object", + "properties": { + "note_attributes": {}, + "location_id": { + "type": ["null", "integer"] + }, + "buyer_accepts_marketing": { + "type": ["null", "boolean"] + }, + "currency": { + "type": ["null", "string"] + }, + "completed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "token": { + "type": ["null", "string"] + }, + "billing_address": { + "type": ["null", "object"], + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "zip": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + } + } + }, + "email": { + "type": ["null", "string"] + }, + "discount_codes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "code": { + "type": ["null", "string"] + } + } + } + }, + "customer_locale": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "gateway": { + "type": ["null", "string"] + }, + "referring_site": { + "type": ["null", "string"] + }, + "source_identifier": { + "type": ["null", "string"] + }, + "total_weight": { + "type": ["null", "integer"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "total_line_items_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "device_id": { + "type": ["null", "integer"] + }, + "phone": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "total_tax": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "subtotal_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "line_items": { + "items": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "source_url": { + "type": ["null", "string"] + }, + "total_discounts": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "note": { + "type": ["null", "string"] + }, + "presentment_currency": { + "type": ["null", "string"] + }, + "shipping_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "applied_discounts": {}, + "custom_tax_lines": {}, + "phone": { + "type": ["null", "string"] + }, + "validation_context": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "carrier_identifier": { + "type": ["null", "string"] + }, + "api_client_id": { + "type": ["null", "integer"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requested_fulfillment_service_id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "carrier_service_id": { + "type": ["null", "integer"] + }, + "delivery_category": { + "type": ["null", "string"] + }, + "markup": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + } + } + } + }, + "user_id": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "shipping_address": { + "type": ["null", "object"], + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "zip": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + } + } + }, + "abandoned_checkout_url": { + "type": ["null", "string"] + }, + "landing_site": { + "type": ["null", "string"] + }, + "customer": { + "type": "object", + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "total_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "cart_token": { + "type": ["null", "string"] + }, + "taxes_included": { + "type": ["null", "boolean"] + } + } + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "metafields", + "json_schema": { + "properties": { + "owner_id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "owner_resource": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "integer", "object", "string"], + "properties": {} + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + }, + "type": "object" + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "custom_collections", + "json_schema": { + "properties": { + "handle": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "string"] + }, + "body_html": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "published_scope": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "image": { + "properties": { + "alt": { + "type": ["null", "string"] + }, + "src": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "height": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "published_at": { + "type": ["null", "string"] + }, + "template_suffix": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "collects", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "collection_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "position": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "sort_value": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "order_refunds", + "json_schema": { + "type": "object", + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "restock": { + "type": ["null", "boolean"] + }, + "order_adjustments": { + "items": { + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "refund_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "kind": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "reason": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "processed_at": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "note": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "refund_line_items": { + "type": ["null", "array"], + "items": { + "properties": { + "location_id": { + "type": ["null", "integer"] + }, + "subtotal_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "total_tax_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "line_item_id": { + "type": ["null", "integer"] + }, + "total_tax": { + "type": ["null", "number"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "line_item": { + "properties": { + "gift_card": { + "type": ["null", "boolean"] + }, + "price": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "properties": { + "price_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + } + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "properties": { + "type": ["null", "array"], + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "quantity": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + }, + "grams": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "price_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "string"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "total_discount_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "discount_allocations": { + "type": ["null", "array"], + "items": { + "properties": { + "amount": { + "type": ["null", "string"] + }, + "amount_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "discount_application_index": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + } + }, + "pre_tax_price_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "total_discount": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "subtotal": { + "type": ["null", "number"] + }, + "restock_type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + } + } + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "order_risks", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "checkout_id": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "score": { + "type": ["null", "string"] + }, + "recommendation": { + "type": ["null", "string"] + }, + "display": { + "type": ["null", "boolean"] + }, + "cause_cancel": { + "type": ["null", "boolean"] + }, + "message": { + "type": ["null", "string"] + }, + "merchant_message": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "transactions", + "json_schema": { + "properties": { + "error_code": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "integer"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "integer"] + }, + "test": { + "type": ["null", "boolean"] + }, + "kind": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "authorization": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "payment_details": { + "properties": { + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_bin": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + }, + "avs_result_code": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "gateway": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "fee_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "gross_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + } + }, + "patternProperties": { + ".+": {} + } + }, + "location_id": { + "type": ["null", "integer"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json new file mode 100644 index 000000000000..8f56a9467f9f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "shop": "SHOP_NAME", + "api_key": "API_KEY", + "api_password": "API_PASSWORD" +} diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/no_refunds_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/no_refunds_catalog.json new file mode 100644 index 000000000000..f42b94664de5 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/no_refunds_catalog.json @@ -0,0 +1,3109 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": { + "type": "object", + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "orders", + "json_schema": { + "properties": { + "presentment_currency": { + "type": ["null", "string"] + }, + "subtotal_price_set": {}, + "total_discounts_set": {}, + "total_line_items_price_set": {}, + "total_price_set": {}, + "total_shipping_price_set": {}, + "total_tax_set": {}, + "total_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "line_items": { + "items": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "processing_method": { + "type": ["null", "string"] + }, + "order_number": { + "type": ["null", "integer"] + }, + "confirmed": { + "type": ["null", "boolean"] + }, + "total_discounts": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "total_line_items_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "order_adjustments": { + "items": { + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "refund_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "kind": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "reason": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "shipping_lines": { + "items": { + "properties": { + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "phone": { + "type": ["null", "string"] + }, + "discounted_price_set": {}, + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "delivery_category": { + "type": ["null", "string"] + }, + "discounted_price": { + "type": ["null", "number"] + }, + "code": { + "type": ["null", "string"] + }, + "requested_fulfillment_service_id": { + "type": ["null", "string"] + }, + "carrier_identifier": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "integer"] + }, + "cancel_reason": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "payment_gateway_names": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "source_identifier": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "processed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "referring_site": { + "type": ["null", "string"] + }, + "contact_email": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "fulfillments": { + "items": { + "properties": { + "location_id": { + "type": ["null", "integer"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "testcase": { + "type": ["null", "boolean"] + }, + "authorization": { + "type": ["null", "string"] + } + } + }, + "tracking_number": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "shipment_status": { + "type": ["null", "string"] + }, + "line_items": { + "items": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "tracking_url": { + "type": ["null", "string"] + }, + "service": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "tracking_urls": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "tracking_numbers": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "id": { + "type": ["null", "integer"] + }, + "tracking_company": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "customer": { + "type": "object", + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "test": { + "type": ["null", "boolean"] + }, + "total_tax": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "payment_details": { + "properties": { + "avs_result_code": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + }, + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_bin": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "number": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "landing_site_ref": { + "type": ["null", "string"] + }, + "shipping_address": { + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + }, + "address2": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "total_price_usd": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "discount_applications": { + "items": { + "properties": { + "target_type": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "target_selection": { + "type": ["null", "string"] + }, + "allocation_method": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "name": { + "type": ["null", "string"] + }, + "note": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "source_url": { + "type": ["null", "string"] + }, + "subtotal_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "billing_address": { + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + }, + "address2": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "landing_site": { + "type": ["null", "string"] + }, + "taxes_included": { + "type": ["null", "boolean"] + }, + "token": { + "type": ["null", "string"] + }, + "app_id": { + "type": ["null", "integer"] + }, + "total_tip_received": { + "type": ["null", "string"] + }, + "browser_ip": { + "type": ["null", "string"] + }, + "discount_codes": { + "items": { + "properties": { + "code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "phone": { + "type": ["null", "string"] + }, + "note_attributes": { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "order_status_url": { + "type": ["null", "string"] + }, + "client_details": { + "properties": { + "session_hash": { + "type": ["null", "string"] + }, + "accept_language": { + "type": ["null", "string"] + }, + "browser_width": { + "type": ["null", "integer"] + }, + "user_agent": { + "type": ["null", "string"] + }, + "browser_ip": { + "type": ["null", "string"] + }, + "browser_height": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "buyer_accepts_marketing": { + "type": ["null", "boolean"] + }, + "checkout_token": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "financial_status": { + "type": ["null", "string"] + }, + "customer_locale": { + "type": ["null", "string"] + }, + "checkout_id": { + "type": ["null", "integer"] + }, + "total_weight": { + "type": ["null", "integer"] + }, + "gateway": { + "type": ["null", "string"] + }, + "cart_token": { + "type": ["null", "string"] + }, + "cancelled_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "refunds": { + "items": { + "properties": { + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "refund_line_items": { + "items": { + "properties": { + "line_item": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "line_item_id": { + "type": ["null", "integer"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "total_tax": { + "type": ["null", "number"] + }, + "restock_type": { + "type": ["null", "string"] + }, + "subtotal": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "restock": { + "type": ["null", "boolean"] + }, + "note": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "processed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "order_adjustments": { + "items": { + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "refund_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "kind": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "reason": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "reference": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "products", + "json_schema": { + "properties": { + "published_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "published_scope": { + "type": ["null", "string"] + }, + "vendor": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "body_html": { + "type": ["null", "string"] + }, + "product_type": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "options": { + "type": ["null", "array"], + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "values": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "id": { + "type": ["null", "integer"] + }, + "position": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + } + }, + "image": { + "properties": { + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "variant_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "height": { + "type": ["null", "integer"] + }, + "alt": { + "type": ["null", "string"] + }, + "src": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "handle": { + "type": ["null", "string"] + }, + "images": { + "type": ["null", "array"], + "items": { + "properties": { + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "variant_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "height": { + "type": ["null", "integer"] + }, + "alt": { + "type": ["null", "string"] + }, + "src": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + } + }, + "template_suffix": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "variants": { + "type": ["null", "array"], + "items": { + "properties": { + "barcode": { + "type": ["null", "string"] + }, + "tax_code": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "weight_unit": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "position": { + "type": ["null", "integer"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "image_id": { + "type": ["null", "integer"] + }, + "inventory_policy": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "inventory_item_id": { + "type": ["null", "integer"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "weight": { + "type": ["null", "number"] + }, + "inventory_management": { + "type": ["null", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "option1": { + "type": ["null", "string"] + }, + "compare_at_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "option2": { + "type": ["null", "string"] + }, + "old_inventory_quantity": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "inventory_quantity": { + "type": ["null", "integer"] + }, + "grams": { + "type": ["null", "integer"] + }, + "option3": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "abandoned_checkouts", + "json_schema": { + "type": "object", + "properties": { + "note_attributes": {}, + "location_id": { + "type": ["null", "integer"] + }, + "buyer_accepts_marketing": { + "type": ["null", "boolean"] + }, + "currency": { + "type": ["null", "string"] + }, + "completed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "token": { + "type": ["null", "string"] + }, + "billing_address": { + "type": ["null", "object"], + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "zip": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + } + } + }, + "email": { + "type": ["null", "string"] + }, + "discount_codes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "code": { + "type": ["null", "string"] + } + } + } + }, + "customer_locale": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "gateway": { + "type": ["null", "string"] + }, + "referring_site": { + "type": ["null", "string"] + }, + "source_identifier": { + "type": ["null", "string"] + }, + "total_weight": { + "type": ["null", "integer"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "total_line_items_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "device_id": { + "type": ["null", "integer"] + }, + "phone": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "total_tax": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "subtotal_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "line_items": { + "items": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "source_url": { + "type": ["null", "string"] + }, + "total_discounts": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "note": { + "type": ["null", "string"] + }, + "presentment_currency": { + "type": ["null", "string"] + }, + "shipping_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "applied_discounts": {}, + "custom_tax_lines": {}, + "phone": { + "type": ["null", "string"] + }, + "validation_context": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "carrier_identifier": { + "type": ["null", "string"] + }, + "api_client_id": { + "type": ["null", "integer"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requested_fulfillment_service_id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "carrier_service_id": { + "type": ["null", "integer"] + }, + "delivery_category": { + "type": ["null", "string"] + }, + "markup": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + } + } + } + }, + "user_id": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "shipping_address": { + "type": ["null", "object"], + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "zip": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + } + } + }, + "abandoned_checkout_url": { + "type": ["null", "string"] + }, + "landing_site": { + "type": ["null", "string"] + }, + "customer": { + "type": "object", + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "total_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "cart_token": { + "type": ["null", "string"] + }, + "taxes_included": { + "type": ["null", "boolean"] + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "metafields", + "json_schema": { + "properties": { + "owner_id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "owner_resource": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "integer", "object", "string"], + "properties": {} + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + }, + "type": "object" + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "custom_collections", + "json_schema": { + "properties": { + "handle": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "string"] + }, + "body_html": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "published_scope": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "image": { + "properties": { + "alt": { + "type": ["null", "string"] + }, + "src": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "height": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "published_at": { + "type": ["null", "string"] + }, + "template_suffix": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "collects", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "collection_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "position": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "sort_value": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "transactions", + "json_schema": { + "properties": { + "error_code": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "integer"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "integer"] + }, + "test": { + "type": ["null", "boolean"] + }, + "kind": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "authorization": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "payment_details": { + "properties": { + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_bin": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + }, + "avs_result_code": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "gateway": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "fee_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "gross_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + } + }, + "patternProperties": { + ".+": {} + } + }, + "location_id": { + "type": ["null", "integer"] + } + }, + "type": "object" + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "incremental", + "cursor_field": ["id"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json new file mode 100644 index 000000000000..3d51ab878498 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json @@ -0,0 +1,32 @@ +{ + "orders": { + "id": 2497723957406 + }, + "collects": { + "id": 17279451103367 + }, + "products": { + "id": 5059311534215 + }, + "customers": { + "id": 3664773775518 + }, + "metafields": { + "id": 11269168529474 + }, + "transactions": { + "id": 3125579481246 + }, + "order_refunds": { + "id": 639978373255 + }, + "custom_collections": { + "id": 153276284994 + }, + "abandoned_checkouts": { + "id": 14254866366622 + }, + "order_risks": { + "id": 5933004390558 + } +} diff --git a/airbyte-integrations/connectors/source-shopify/main.py b/airbyte-integrations/connectors/source-shopify/main.py new file mode 100644 index 000000000000..2caa48357846 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_shopify import SourceShopify + +if __name__ == "__main__": + source = SourceShopify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-shopify/requirements.txt b/airbyte-integrations/connectors/source-shopify/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-shopify/setup.py b/airbyte-integrations/connectors/source-shopify/setup.py new file mode 100644 index 000000000000..9f1f4ac599e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/setup.py @@ -0,0 +1,48 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_shopify", + description="Source CDK implementation for Shopify.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/__init__.py b/airbyte-integrations/connectors/source-shopify/source_shopify/__init__.py new file mode 100644 index 000000000000..1d878edae29f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2021 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourceShopify + +__all__ = ["SourceShopify"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/TODO.md b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json new file mode 100644 index 000000000000..34e31847c7f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json @@ -0,0 +1,770 @@ +{ + "type": "object", + "properties": { + "note_attributes": {}, + "location_id": { + "type": ["null", "integer"] + }, + "buyer_accepts_marketing": { + "type": ["null", "boolean"] + }, + "currency": { + "type": ["null", "string"] + }, + "completed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "token": { + "type": ["null", "string"] + }, + "billing_address": { + "type": ["null", "object"], + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "zip": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + } + } + }, + "email": { + "type": ["null", "string"] + }, + "discount_codes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "code": { + "type": ["null", "string"] + } + } + } + }, + "customer_locale": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "gateway": { + "type": ["null", "string"] + }, + "referring_site": { + "type": ["null", "string"] + }, + "source_identifier": { + "type": ["null", "string"] + }, + "total_weight": { + "type": ["null", "integer"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "total_line_items_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "device_id": { + "type": ["null", "integer"] + }, + "phone": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "total_tax": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "subtotal_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "line_items": { + "items": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "source_url": { + "type": ["null", "string"] + }, + "total_discounts": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "note": { + "type": ["null", "string"] + }, + "presentment_currency": { + "type": ["null", "string"] + }, + "shipping_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "applied_discounts": {}, + "custom_tax_lines": {}, + "phone": { + "type": ["null", "string"] + }, + "validation_context": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "carrier_identifier": { + "type": ["null", "string"] + }, + "api_client_id": { + "type": ["null", "integer"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requested_fulfillment_service_id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "carrier_service_id": { + "type": ["null", "integer"] + }, + "delivery_category": { + "type": ["null", "string"] + }, + "markup": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + } + } + } + }, + "user_id": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "shipping_address": { + "type": ["null", "object"], + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "zip": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + } + } + }, + "abandoned_checkout_url": { + "type": ["null", "string"] + }, + "landing_site": { + "type": ["null", "string"] + }, + "customer": { + "type": "object", + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "total_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "cart_token": { + "type": ["null", "string"] + }, + "taxes_included": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collects.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collects.json new file mode 100644 index 000000000000..aa2bd0a282e3 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/collects.json @@ -0,0 +1,28 @@ +{ + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "collection_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "position": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "sort_value": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json new file mode 100644 index 000000000000..82ec595787d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json @@ -0,0 +1,55 @@ +{ + "properties": { + "handle": { + "type": ["null", "string"] + }, + "sort_order": { + "type": ["null", "string"] + }, + "body_html": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "published_scope": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "image": { + "properties": { + "alt": { + "type": ["null", "string"] + }, + "src": { + "type": ["null", "string"] + }, + "width": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "height": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "published_at": { + "type": ["null", "string"] + }, + "template_suffix": { + "type": ["null", "string"] + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json new file mode 100644 index 000000000000..648b89423bb1 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/customers.json @@ -0,0 +1,196 @@ +{ + "type": "object", + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafields.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafields.json new file mode 100644 index 000000000000..54102d3951a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafields.json @@ -0,0 +1,41 @@ +{ + "properties": { + "owner_id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "owner_resource": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "namespace": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "integer", "object", "string"], + "properties": {} + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json new file mode 100644 index 000000000000..3aa2da402e54 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json @@ -0,0 +1,395 @@ +{ + "type": "object", + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "restock": { + "type": ["null", "boolean"] + }, + "order_adjustments": { + "items": { + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "refund_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "kind": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "reason": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "processed_at": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "note": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "refund_line_items": { + "type": ["null", "array"], + "items": { + "properties": { + "location_id": { + "type": ["null", "integer"] + }, + "subtotal_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "total_tax_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "line_item_id": { + "type": ["null", "integer"] + }, + "total_tax": { + "type": ["null", "number"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "line_item": { + "properties": { + "gift_card": { + "type": ["null", "boolean"] + }, + "price": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "properties": { + "price_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + } + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "properties": { + "type": ["null", "array"], + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "quantity": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + }, + "grams": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "price_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "string"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "total_discount_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "discount_allocations": { + "type": ["null", "array"], + "items": { + "properties": { + "amount": { + "type": ["null", "string"] + }, + "amount_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "discount_application_index": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + } + }, + "pre_tax_price_set": { + "properties": { + "shop_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "presentment_money": { + "properties": { + "currency_code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + }, + "type": ["null", "object"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "total_discount": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "subtotal": { + "type": ["null", "number"] + }, + "restock_type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json new file mode 100644 index 000000000000..61cfde09f464 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json @@ -0,0 +1,35 @@ +{ + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "checkout_id": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "score": { + "type": ["null", "string"] + }, + "recommendation": { + "type": ["null", "string"] + }, + "display": { + "type": ["null", "boolean"] + }, + "cause_cancel": { + "type": ["null", "boolean"] + }, + "message": { + "type": ["null", "string"] + }, + "merchant_message": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json new file mode 100644 index 000000000000..a6703945f518 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json @@ -0,0 +1,1426 @@ +{ + "properties": { + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "app_id": { + "type": ["null", "integer"] + }, + "browser_ip": { + "type": ["null", "string"] + }, + "buyer_accepts_marketing": { + "type": ["null", "boolean"] + }, + "cancel_reason": { + "type": ["null", "string"] + }, + "cancelled_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "cart_token": { + "type": ["null", "string"] + }, + "checkout_id": { + "type": ["null", "integer"] + }, + "checkout_token": { + "type": ["null", "string"] + }, + "client_details": { + "type": ["null", "object"], + "properties": { + "accept_language": { + "type": ["null", "string"] + }, + "browser_height": { + "type": ["null", "integer"] + }, + "browser_ip": { + "type": ["null", "string"] + }, + "browser_width": { + "type": ["null", "integer"] + }, + "session_hash": { + "type": ["null", "string"] + }, + "user_agent": { + "type": ["null", "string"] + } + } + }, + "closed_at": { + "type": ["null", "string"] + }, + "confirmed": { + "type": ["null", "boolean"] + }, + "contact_email": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "current_subtotal_price": { + "type": ["null", "string"] + }, + "current_subtotal_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "current_total_discounts": { + "type": ["null", "string"] + }, + "current_total_discounts_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "current_total_duties_set": { + "type": ["null", "string"] + }, + "current_total_price": { + "type": ["null", "string"] + }, + "current_total_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "current_total_tax": { + "type": ["null", "string"] + }, + "current_total_tax_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "customer_locale": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "string"] + }, + "discount_codes": { + "type": ["null", "array"] + }, + "email": { + "type": ["null", "string"] + }, + "financial_status": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gateway": { + "type": ["null", "string"] + }, + "landing_site": { + "type": ["null", "string"] + }, + "landing_site_ref": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "note": { + "type": ["null", "string"] + }, + "note_attributes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "number": { + "type": ["null", "integer"] + }, + "order_number": { + "type": ["null", "integer"] + }, + "order_status_url": { + "type": ["null", "string"] + }, + "original_total_duties_set": { + "type": ["null", "string"] + }, + "payment_gateway_names": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "phone": { + "type": ["null", "string"] + }, + "presentment_currency": { + "type": ["null", "string"] + }, + "processed_at": { + "type": ["null", "string"] + }, + "processing_method": { + "type": ["null", "string"] + }, + "reference": { + "type": ["null", "string"] + }, + "referring_site": { + "type": ["null", "string"] + }, + "source_identifier": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "source_url": { + "type": ["null", "string"] + }, + "subtotal_price": { + "type": ["null", "string"] + }, + "subtotal_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "tags": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "price": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + } + } + } + }, + "taxes_included": { + "type": ["null", "boolean"] + }, + "test": { + "type": ["null", "boolean"] + }, + "token": { + "type": ["null", "string"] + }, + "total_discounts": { + "type": ["null", "string"] + }, + "total_discounts_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "total_line_items_price": { + "type": ["null", "string"] + }, + "total_line_items_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "total_outstanding": { + "type": ["null", "string"] + }, + "total_price": { + "type": ["null", "string"] + }, + "total_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "total_price_usd": { + "type": ["null", "string"] + }, + "total_shipping_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "total_tax": { + "type": ["null", "string"] + }, + "total_tax_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "total_tip_received": { + "type": ["null", "string"] + }, + "total_weight": { + "type": ["null", "integer"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "string"] + }, + "billing_address": { + "type": ["null", "object"], + "properties": { + "first_name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + } + } + }, + "customer": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "note": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "phone": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "accepts_marketing_updated_at": { + "type": ["null", "string"] + }, + "marketing_opt_in_level": { + "type": ["null", "string"] + }, + "tax_exemptions": { + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "country_name": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "boolean"] + } + } + } + } + }, + "discount_applications": { + "type": ["null", "array"] + }, + "fulfillments": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "receipt": { + "type": ["null", "object"] + }, + "service": { + "type": ["null", "string"] + }, + "shipment_status": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "tracking_company": { + "type": ["null", "string"] + }, + "tracking_number": { + "type": ["null", "string"] + }, + "tracking_numbers": { + "type": ["null", "array"] + }, + "tracking_url": { + "type": ["null", "string"] + }, + "tracking_urls": { + "type": ["null", "array"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "line_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "destination_location": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + } + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "grams": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "origin_location": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + } + }, + "price": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "properties": { + "type": ["null", "array"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "sku": { + "type": ["null", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + }, + "total_discount": { + "type": ["null", "string"] + }, + "total_discount_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "variant_id": { + "type": ["null", "integer"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "price": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "rate": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + } + } + } + }, + "duties": { + "type": ["null", "array"] + }, + "discount_allocations": { + "type": ["null", "array"] + } + } + } + } + } + } + }, + "line_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "destination_location": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + } + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "grams": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "origin_location": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + } + }, + "price": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "properties": { + "type": ["null", "array"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "sku": { + "type": ["null", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + }, + "total_discount": { + "type": ["null", "string"] + }, + "total_discount_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "variant_id": { + "type": ["null", "integer"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "price": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "rate": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + } + } + } + }, + "duties": { + "type": ["null", "array"] + }, + "discount_allocations": { + "type": ["null", "array"] + } + } + } + }, + "payment_details": { + "type": ["null", "object"], + "properties": { + "credit_card_bin": { + "type": ["null", "string"] + }, + "avs_result_code": { + "type": ["null", "string"] + }, + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + } + } + }, + "refunds": { + "type": ["null", "array"] + }, + "shipping_address": { + "type": ["null", "object"], + "properties": { + "first_name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + } + } + }, + "shipping_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "carrier_identifier": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "delivery_category": { + "type": ["null", "string"] + }, + "discounted_price": { + "type": ["null", "string"] + }, + "discounted_price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "phone": { + "type": ["null", "string"] + }, + "price": { + "type": ["null", "string"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "requested_fulfillment_service_id": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "tax_lines": { + "type": ["null", "array"] + }, + "discount_allocations": { + "type": ["null", "array"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json new file mode 100644 index 000000000000..f7cbc9f00cec --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json @@ -0,0 +1,1581 @@ +{ + "properties": { + "presentment_currency": { + "type": ["null", "string"] + }, + "subtotal_price_set": {}, + "total_discounts_set": {}, + "total_line_items_price_set": {}, + "total_price_set": {}, + "total_shipping_price_set": {}, + "total_tax_set": {}, + "total_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "line_items": { + "items": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "processing_method": { + "type": ["null", "string"] + }, + "order_number": { + "type": ["null", "integer"] + }, + "confirmed": { + "type": ["null", "boolean"] + }, + "total_discounts": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "total_line_items_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "order_adjustments": { + "items": { + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "refund_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "kind": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "reason": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "shipping_lines": { + "items": { + "properties": { + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "phone": { + "type": ["null", "string"] + }, + "discounted_price_set": {}, + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "delivery_category": { + "type": ["null", "string"] + }, + "discounted_price": { + "type": ["null", "number"] + }, + "code": { + "type": ["null", "string"] + }, + "requested_fulfillment_service_id": { + "type": ["null", "string"] + }, + "carrier_identifier": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "integer"] + }, + "cancel_reason": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "payment_gateway_names": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "source_identifier": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "processed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "referring_site": { + "type": ["null", "string"] + }, + "contact_email": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "fulfillments": { + "items": { + "properties": { + "location_id": { + "type": ["null", "integer"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "testcase": { + "type": ["null", "boolean"] + }, + "authorization": { + "type": ["null", "string"] + } + } + }, + "tracking_number": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "shipment_status": { + "type": ["null", "string"] + }, + "line_items": { + "items": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "tracking_url": { + "type": ["null", "string"] + }, + "service": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "tracking_urls": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "tracking_numbers": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "id": { + "type": ["null", "integer"] + }, + "tracking_company": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "customer": { + "type": "object", + "properties": { + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "default_address": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "total_spent": { + "type": ["null", "string"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "first_name": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "note": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "addresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "city": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "country_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "customer_id": { + "type": ["null", "integer"] + }, + "default": { + "type": ["null", "boolean"] + }, + "last_name": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + } + } + } + }, + "last_name": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "accepts_marketing_updated_at": { + "anyOf": [ + { + "type": "string", + "format": "date-time" + }, + { + "type": "string" + }, + { + "type": "null" + } + ] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "test": { + "type": ["null", "boolean"] + }, + "total_tax": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "payment_details": { + "properties": { + "avs_result_code": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + }, + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_bin": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "number": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "landing_site_ref": { + "type": ["null", "string"] + }, + "shipping_address": { + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + }, + "address2": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "total_price_usd": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "discount_applications": { + "items": { + "properties": { + "target_type": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "target_selection": { + "type": ["null", "string"] + }, + "allocation_method": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "name": { + "type": ["null", "string"] + }, + "note": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "source_url": { + "type": ["null", "string"] + }, + "subtotal_price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "billing_address": { + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + }, + "address2": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "landing_site": { + "type": ["null", "string"] + }, + "taxes_included": { + "type": ["null", "boolean"] + }, + "token": { + "type": ["null", "string"] + }, + "app_id": { + "type": ["null", "integer"] + }, + "total_tip_received": { + "type": ["null", "string"] + }, + "browser_ip": { + "type": ["null", "string"] + }, + "discount_codes": { + "items": { + "properties": { + "code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "phone": { + "type": ["null", "string"] + }, + "note_attributes": { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "order_status_url": { + "type": ["null", "string"] + }, + "client_details": { + "properties": { + "session_hash": { + "type": ["null", "string"] + }, + "accept_language": { + "type": ["null", "string"] + }, + "browser_width": { + "type": ["null", "integer"] + }, + "user_agent": { + "type": ["null", "string"] + }, + "browser_ip": { + "type": ["null", "string"] + }, + "browser_height": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "buyer_accepts_marketing": { + "type": ["null", "boolean"] + }, + "checkout_token": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "financial_status": { + "type": ["null", "string"] + }, + "customer_locale": { + "type": ["null", "string"] + }, + "checkout_id": { + "type": ["null", "integer"] + }, + "total_weight": { + "type": ["null", "integer"] + }, + "gateway": { + "type": ["null", "string"] + }, + "cart_token": { + "type": ["null", "string"] + }, + "cancelled_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "refunds": { + "items": { + "properties": { + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "refund_line_items": { + "items": { + "properties": { + "line_item": { + "properties": { + "applied_discounts": {}, + "total_discount_set": {}, + "pre_tax_price_set": {}, + "price_set": {}, + "grams": { + "type": ["null", "integer"] + }, + "compare_at_price": { + "type": ["null", "string"] + }, + "destination_location_id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "line_price": { + "type": ["null", "string"] + }, + "origin_location_id": { + "type": ["null", "integer"] + }, + "applied_discount": { + "type": ["null", "integer"] + }, + "fulfillable_quantity": { + "type": ["null", "integer"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "properties": { + "anyOf": [ + { + "items": { + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + { + "properties": {}, + "type": ["null", "object"] + } + ] + }, + "tax_code": { + "type": ["null", "string"] + }, + "discount_allocations": { + "items": { + "properties": { + "discount_application_index": { + "type": ["null", "integer"] + }, + "amount_set": {}, + "amount": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "pre_tax_price": { + "type": ["null", "number"] + }, + "sku": { + "type": ["null", "string"] + }, + "product_exists": { + "type": ["null", "boolean"] + }, + "total_discount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "name": { + "type": ["null", "string"] + }, + "fulfillment_status": { + "type": ["null", "string"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "integer", "string"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "vendor": { + "type": ["null", "string"] + }, + "tax_lines": { + "items": { + "properties": { + "price_set": {}, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "title": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "compare_at": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "source": { + "type": ["null", "string"] + }, + "zone": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "origin_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "price": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "variant_inventory_management": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "destination_location": { + "properties": { + "country_code": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "address2": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "line_item_id": { + "type": ["null", "integer"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "integer"] + }, + "total_tax": { + "type": ["null", "number"] + }, + "restock_type": { + "type": ["null", "string"] + }, + "subtotal": { + "type": ["null", "number"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "restock": { + "type": ["null", "boolean"] + }, + "note": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "processed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "order_adjustments": { + "items": { + "properties": { + "order_id": { + "type": ["null", "integer"] + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "refund_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "kind": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "reason": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "reference": { + "type": ["null", "string"] + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json new file mode 100644 index 000000000000..40fcd0ee11f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json @@ -0,0 +1,100 @@ +{ + "properties": { + "error_code": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "integer"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "integer"] + }, + "test": { + "type": ["null", "boolean"] + }, + "kind": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "authorization": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "source_name": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "payment_details": { + "properties": { + "cvv_result_code": { + "type": ["null", "string"] + }, + "credit_card_bin": { + "type": ["null", "string"] + }, + "credit_card_company": { + "type": ["null", "string"] + }, + "credit_card_number": { + "type": ["null", "string"] + }, + "avs_result_code": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "gateway": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "fee_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "gross_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + } + }, + "patternProperties": { + ".+": {} + } + }, + "location_id": { + "type": ["null", "integer"] + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py new file mode 100644 index 000000000000..d4dcc6f31e96 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -0,0 +1,263 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from abc import ABC, abstractmethod +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator + + +class ShopifyStream(HttpStream, ABC): + + # Latest Stable Release + api_version = "2021-04" + # Page size + limit = 250 + # Define primary key to all streams as primary key, sort key + primary_key = "id" + + def __init__(self, shop: str, start_date: str, api_password: str, **kwargs): + super().__init__(**kwargs) + self.start_date = start_date + self.shop = shop + self.api_password = api_password + self.since_id = 0 + + @property + def url_base(self) -> str: + return f"https://{self.shop}.myshopify.com/admin/api/{self.api_version}/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + decoded_response = response.json() + if len(decoded_response.get(self.data_field)) < self.limit: + return None + else: + self.since_id = decoded_response.get(self.data_field)[-1]["id"] + return {"since_id": self.since_id} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = {"limit": self.limit, "order": f"{self.primary_key} asc", "created_at_min": self.start_date} + if next_page_token: + params.pop("created_at_min", None) + params.pop("order", None) + params.update(**next_page_token) + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + json_response = response.json() + records = json_response.get(self.data_field, []) if self.data_field is not None else json_response + yield from records + + @property + @abstractmethod + def data_field(self) -> str: + """The name of the field in the response which contains the data""" + + +# Basic incremental stream +class IncrementalShopifyStream(ShopifyStream, ABC): + + # Getting page size as 'limit' from parrent class + @property + def limit(self): + return super().limit + + state_checkpoint_interval = limit + + cursor_field = "id" + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + return {self.cursor_field: max(latest_record.get(self.cursor_field, 0), current_stream_state.get(self.cursor_field, 0))} + + def request_params(self, stream_state=None, **kwargs): + stream_state = stream_state or {} + params = super().request_params(stream_state=stream_state, **kwargs) + params["since_id"] = stream_state.get(self.cursor_field) + return params + + +class Customers(IncrementalShopifyStream): + data_field = "customers" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + +class Orders(IncrementalShopifyStream): + data_field = "orders" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json?status=any" + + +class Products(IncrementalShopifyStream): + data_field = "products" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + +class AbandonedCheckouts(IncrementalShopifyStream): + data_field = "checkouts" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json?status=any" + + +class Metafields(IncrementalShopifyStream): + data_field = "metafields" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + def request_params(self, stream_state=None, **kwargs) -> MutableMapping[str, Any]: + stream_state = stream_state or {} + params = {"limit": self.limit} + params["since_id"] = stream_state.get(self.cursor_field) + return params + + +class CustomCollections(IncrementalShopifyStream): + data_field = "custom_collections" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + +class Collects(IncrementalShopifyStream): + data_field = "collects" + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + def request_params(self, stream_state=None, **kwargs) -> MutableMapping[str, Any]: + stream_state = stream_state or {} + params = {"limit": self.limit} + params["since_id"] = stream_state.get(self.cursor_field) + return params + + +class OrderRefunds(IncrementalShopifyStream): + data_field = "refunds" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + order_id = stream_slice["order_id"] + return f"orders/{order_id}/{self.data_field}.json" + + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + orders_stream = Orders(authenticator=self.authenticator, shop=self.shop, start_date=self.start_date, api_password=self.api_password) + for data in orders_stream.read_records(sync_mode=SyncMode.full_refresh): + yield from super().read_records(stream_slice={"order_id": data["id"]}, **kwargs) + + +class OrderRisks(IncrementalShopifyStream): + data_field = "risks" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + order_id = stream_slice["order_id"] + return f"orders/{order_id}/{self.data_field}.json" + + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + orders_stream = Orders(authenticator=self.authenticator, shop=self.shop, start_date=self.start_date, api_password=self.api_password) + for data in orders_stream.read_records(sync_mode=SyncMode.full_refresh): + yield from super().read_records(stream_slice={"order_id": data["id"]}, **kwargs) + + +class Transactions(IncrementalShopifyStream): + data_field = "transactions" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + order_id = stream_slice["order_id"] + return f"orders/{order_id}/{self.data_field}.json" + + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + orders_stream = Orders(authenticator=self.authenticator, shop=self.shop, start_date=self.start_date, api_password=self.api_password) + for data in orders_stream.read_records(sync_mode=SyncMode.full_refresh): + yield from super().read_records(stream_slice={"order_id": data["id"]}, **kwargs) + + +class ShopifyAuthenticator(HttpAuthenticator): + + """ + Making Authenticator to be able to accept Header-Based authentication. + """ + + def __init__(self, token: str): + self.token = token + + def get_auth_header(self) -> Mapping[str, Any]: + return {"X-Shopify-Access-Token": f"{self.token}"} + + +# Basic Connections Check +class SourceShopify(AbstractSource): + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: + + """ + Testing connection availability for the connector. + """ + + shop = config["shop"] + api_pass = config["api_password"] + api_version = "2021-04" # Latest Stable Release + + headers = {"X-Shopify-Access-Token": api_pass} + url = f"https://{shop}.myshopify.com/admin/api/{api_version}/shop.json" + + try: + session = requests.get(url, headers=headers) + session.raise_for_status() + return True, None + except requests.exceptions.RequestException as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + + """ + Mapping a input config of the user input configuration as defined in the connector spec. + Defining streams to run. + """ + + auth = ShopifyAuthenticator(token=config["api_password"]) + args = {"authenticator": auth, "shop": config["shop"], "start_date": config["start_date"], "api_password": config["api_password"]} + return [ + Customers(**args), + Orders(**args), + Products(**args), + AbandonedCheckouts(**args), + Metafields(**args), + CustomCollections(**args), + Collects(**args), + OrderRefunds(**args), + OrderRisks(**args), + Transactions(**args), + ] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json new file mode 100644 index 000000000000..3af46170f8de --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json @@ -0,0 +1,27 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Shopify Source CDK Specifications", + "type": "object", + "required": ["shop", "start_date", "api_password"], + "additionalProperties": false, + "properties": { + "shop": { + "type": "string", + "description": "The name of the shopify store. For https://EXAMPLE.myshopify.com, the shop name is 'EXAMPLE'." + }, + "start_date": { + "type": "string", + "description": "The date you would like to replicate data. Format: YYYY-MM-DD.", + "examples": ["2021-01-01"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + }, + "api_password": { + "type": "string", + "description": "The API PASSWORD for a private application in Shopify shop.", + "airbyte_secret": true + } + } + } +} diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py new file mode 100644 index 000000000000..b8a8150b507f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py @@ -0,0 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +def test_example_method(): + assert True diff --git a/airbyte-migration/src/main/resources/migrations/migrationV0_25_0/NamespaceDefinitionType.yaml b/airbyte-migration/src/main/resources/migrations/migrationV0_25_0/NamespaceDefinitionType.yaml new file mode 100644 index 000000000000..d81a269dfd69 --- /dev/null +++ b/airbyte-migration/src/main/resources/migrations/migrationV0_25_0/NamespaceDefinitionType.yaml @@ -0,0 +1,11 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/NamespaceDefinitionType.yaml +title: Namespace Definition +description: Method used for computing final namespace in destination +type: string +default: source +enum: + - source + - destination + - customformat diff --git a/airbyte-migration/src/main/resources/migrations/migrationV0_25_0/StandardSync.yaml b/airbyte-migration/src/main/resources/migrations/migrationV0_25_0/StandardSync.yaml new file mode 100644 index 000000000000..b980b6834d2c --- /dev/null +++ b/airbyte-migration/src/main/resources/migrations/migrationV0_25_0/StandardSync.yaml @@ -0,0 +1,70 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/resources/types/StandardSync.yaml +title: StandardSync +description: configuration required for sync for ALL taps +type: object +required: + - sourceId + - destinationId + - name + - catalog + - manual +additionalProperties: false +properties: + namespaceDefinition: + "$ref": NamespaceDefinitionType.yaml + namespaceFormat: + type: string + default: null + example: "${SOURCE_NAMESPACE}" + prefix: + description: Prefix that will be prepended to the name of each stream when it is written to the destination. + type: string + sourceId: + type: string + format: uuid + destinationId: + type: string + format: uuid + operationIds: + type: array + items: + type: string + format: uuid + connectionId: + type: string + format: uuid + name: + type: string + catalog: + existingJavaType: io.airbyte.protocol.models.ConfiguredAirbyteCatalog + status: + type: string + enum: + - active + - inactive + - deprecated + # Ideally schedule and manual should be a union, but java + # codegen does not handle the union type properly. + # When schedule is defined, manual should be false. + schedule: + type: object + required: + - timeUnit + - units + additionalProperties: false + properties: + timeUnit: + type: string + enum: + - minutes + - hours + - days + - weeks + - months + units: + type: integer + # When manual is true, schedule should be null, and will be ignored. + manual: + type: boolean diff --git a/docs/troubleshooting/README.md b/docs/troubleshooting/README.md new file mode 100644 index 000000000000..8554af65aade --- /dev/null +++ b/docs/troubleshooting/README.md @@ -0,0 +1,2 @@ +# Troubleshooting + diff --git a/docs/troubleshooting/new-connection.md b/docs/troubleshooting/new-connection.md new file mode 100644 index 000000000000..2cbb1e1bb482 --- /dev/null +++ b/docs/troubleshooting/new-connection.md @@ -0,0 +1,34 @@ +--- +description: Common issues when trying to set up a new connection (source/destination) +--- + +# Setting new connection + +## Onboarding + +### Airbyte is stuck while loading required configuration parameters for my connector + +Example of the issue: + +![](../.gitbook/assets/faq_stuck_onboarding.png) + +To load configuration parameters, Airbyte must first `docker pull` the connector's image, which may be many hundreds of megabytes. Under poor connectivity conditions, the request to pull the image may take a very long time or time out. More context on this issue can be found [here](https://github.com/airbytehq/airbyte/issues/1462). If your Internet speed is less than 30Mbps down or are running bandwidth-consuming workloads concurrently with Airbyte, you may encounter this issue. Run a [speed test](https://fast.com/) to verify your internet speed. + +One workaround is to manually pull the latest version of every connector you'll use then resetting Airbyte. Note that this will remove any configured connections, sources, or destinations you currently have in Airbyte. To do this: + +1. Decide which connectors you'd like to use. For this example let's say you want the Postgres source and the Snowflake destination. +2. Find the Docker image name of those connectors. Look [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-config/init/src/main/resources/seed/source_definitions.yaml) for sources and [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml) for destinations. For each of the connectors you'd like to use, copy the value of the `dockerRepository` and `dockerImageTag` fields. For example, for the Postgres source this would be `airbyte/source-postgres` and e.g `0.1.6`. +3. For **each of the connectors** you'd like to use, from your shell run `docker pull :`, replacing `` and `` with the values copied from the step above e.g: `docker pull airbyte/source-postgres:0.1.6`. +4. Once you've finished downloading all the images, from the Airbyte repository root run `docker-compose down -v` followed by `docker-compose up`. +5. The issue should be resolved. + +If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). + +### **Connection refused errors when connecting to a local db** + +Depending on your Docker network configuration, you may not be able to connect to `localhost` or `127.0.0.1` directly. + +If you are running into connection refused errors when running Airbyte via Docker Compose on Mac, try using `host.docker.internal` as the host. On Linux, you may have to modify `docker-compose.yml` and add a host that maps to your local machine using [`extra_hosts`](https://docs.docker.com/compose/compose-file/compose-file-v3/#extra_hosts). + +### I don’t see a form when selecting a connector +We’ve had that issue once. (no spinner & 500 http error). We don’t know why. Resolution: try to stop airbyte (`docker-compose down`) & restart (`docker-compose up`) diff --git a/docs/troubleshooting/on-deploy.md b/docs/troubleshooting/on-deploy.md new file mode 100644 index 000000000000..10783e8e3163 --- /dev/null +++ b/docs/troubleshooting/on-deploy.md @@ -0,0 +1,69 @@ +--- +description: Common issues and their workarounds related when trying launch Airbyte +--- + +# On deployment + +## Stuck in onboarding, can’t skip or do anything +To full reset Airbyte, you need to also delete the docker volumes associated with Airbyte. This is where data is stored. +Assuming that you are running Airbyte by running `docker-compose up`, then what you need to do is: +* Turn off Airbyte completely: `docker-compose down -v` +* Turn Airbyte back on: `docker-compose up` + +that should handle you getting reset to the beginning. +I would be curious if we can see the logs associated with the failure you are seeing. I would say if after you reset you run into it again we can debug that. + + +### I have run `docker-compose up` and can not access the interface + +- If you see a blank screen and not a loading icon: + +Check your web browser version; Some old versions of web browsers doesn't support our current Front-end stack. + +- If you see a loading icon or the message `Cannot reach the server` persist: + +Check if all Airbyte containers are running, executing: `docker ps` + +```text +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +f45f3cfe1e16 airbyte/scheduler:1.11.1-alpha "/bin/bash -c './wai…" 2 hours ago Up 2 hours airbyte-scheduler +f02fc709b130 airbyte/server:1.11.1-alpha "/bin/bash -c './wai…" 2 hours ago Up 2 hours 8000/tcp, [...] :::8001->8001/tcp airbyte-server +153b2b322870 airbyte/webapp:1.11.1-alpha "/docker-entrypoint.…" 2 hours ago Up 2 hours :::8000->80/tcp airbyte-webapp +b88d94652268 airbyte/db:1.11.1-alpha "docker-entrypoint.s…" 2 hours ago Up 2 hours 5432/tcp airbyte-db +0573681a10e0 temporalio/auto-setup:1.7.0 "/entrypoint.sh /bin…" 2 hours ago Up 2 hours 6933-6935/tcp, [...] airbyte-temporal +``` +You must see 5 containers running. If you are not seeing execute the following steps: +* `docker-compose down -v` +* `docker-compose up` +Keep in mind the commands above will delete ALL containers, volumes and data created by Airbyte. + We do not recommend this is you already deploy and have connection created. + +First, let's check the server logs by running `docker logs airbyte-server | grep ERROR`.
    +If this command returns any output, please run `docker logs airbyte-server > airbyte-server.log`.
    +This command will create a file in the current directory. We advise you to send a message on our #issues on Slack channel + +If you don't have any server errors let's check the scheduler, `docker logs airbyte-scheduler | grep ERROR`.
    +If this command returns any output, please run `docker logs airbyte-scheduler > airbyte-scheduler.log`.
    +This command will create a file in the current directory. We advise you to send a message on our #issues on Slack channel + +If there is no error printed in both cases, we recommend running: `docker restart airbyte-server airbyte-scheduler`
    +Wait a few moments and try to access the interface again. + +### `docker.errors.DockerException`: Error while fetching server API version + +If you see the following error: + +```text +docker.errors.DockerException: Error while fetching server API +version: ('Connection aborted.', FileNotFoundError(2, 'No such file or +directory')) +``` + +It usually means that Docker isn't running on your machine \(and a running Docker daemon is required to run Airbyte\). An easy way to verify this is to run `docker ps`, which will show `Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?` if the Docker daemon is not running on your machine. + +This happens (sometimes) on Windows system when you first install `docker`. You need to restart your machine. + + +### Getting a weird error related to setting up the Airbyte server when running Docker Compose -- wondering if this is because I played around with Airbyte in a past version? + +If you are okay with losing your previous Airbyte configurations, you can run `docker-compose down -v` and that should fix things then `docker-compose up`. diff --git a/docs/troubleshooting/running-sync.md b/docs/troubleshooting/running-sync.md new file mode 100644 index 000000000000..23ba782110e7 --- /dev/null +++ b/docs/troubleshooting/running-sync.md @@ -0,0 +1,33 @@ +# Syncing a connection + + +### One of your sync jobs is failing + +Several things to check: + +* **Is Airbyte updated to your latest version?** You can see the latest version [here](https://github.com/airbytehq/airbyte/tags). If not, please upgrade to the latest one, [upgrading instructions are here](../tutorials/upgrading-airbyte.md) +* **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-config/init/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. + +If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). + +### Your incremental connection is not working + +Our current version of incremental is [append](../understanding-airbyte/connections/incremental-append.md). It works from a cursor field. So you need to check which cursor field you're using and if it's well populated in every record in your table. + +If this is true, then, there are still several things to check: + +* **Is Airbyte updated to your latest version?** You can see the latest version [here](https://github.com/airbytehq/airbyte/tags). If not, please upgrade to the latest one, [upgrading instructions are here](../tutorials/upgrading-airbyte.md) +* **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-config/init/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. + +If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). + +### **Airbyte says successful sync, but some records are missing** + +Several things to check: + +* What is the name of the table you are looking at in the destination? Let's make sure you're not looking at a temporary table. +* **Is the basic normalization toggle set to true at the connection settings?** If it's false, you won't see columns but most probably a JSON file. So you need to switch it on true, and try again. +* **Is Airbyte updated to your latest version?** You can see the latest version [here](https://github.com/airbytehq/airbyte/tags). If not, please upgrade to the latest one, [upgrading instructions are here](../tutorials/upgrading-airbyte.md) +* **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-config/init/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. + +If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). diff --git a/docs/troubleshooting/upgrading.md b/docs/troubleshooting/upgrading.md new file mode 100644 index 000000000000..e69de29bb2d1 From ba92bc6e3cc77e1353975933cad299aa8adf063b Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Tue, 15 Jun 2021 09:39:53 +0300 Subject: [PATCH 11/19] version bump --- .pre-commit-config.yaml | 21 ------------------- .../39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../source-googleanalytics-singer/Dockerfile | 2 +- .../acceptance-test-config.yml | 17 ++++++++------- .../integration_tests/invalid_config.json | 2 +- .../sample_files/abnormal_state.json | 8 +++---- 7 files changed, 17 insertions(+), 37 deletions(-) delete mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 395cd9b5b172..000000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -repos: - - repo: https://github.com/ambv/black - rev: stable - hooks: - - id: black - args: ["--line-length=140"] - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v1.2.3 - hooks: - - id: flake8 - args: ["--config=tools/python/.flake8"] - - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 - hooks: - - id: isort - args: ["--settings-path=tools/python/.isort.cfg"] - - repo: https://github.com/johann-petrak/licenseheaders.git - rev: 'master' - hooks: - - id: licenseheaders - args: ["--tmpl=LICENSE", "--ext=py", "-f"] diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json index b801cbf26804..69023fb04c98 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1", "name": "Google Analytics", "dockerRepository": "airbyte/source-googleanalytics-singer", - "dockerImageTag": "0.2.3", + "dockerImageTag": "0.2.4", "documentationUrl": "https://hub.docker.com/r/airbyte/source-googleanalytics-singer", "icon": "google-analytics.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5f0403dea55e..d4397ebef362 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -95,7 +95,7 @@ - sourceDefinitionId: 39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1 name: Google Analytics dockerRepository: airbyte/source-googleanalytics-singer - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.4 documentationUrl: https://hub.docker.com/r/airbyte/source-googleanalytics-singer icon: google-analytics.svg - sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile b/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile index d24563359566..48ba0055582c 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile @@ -8,7 +8,7 @@ ENV CODE_PATH="source_googleanalytics_singer" ENV AIRBYTE_IMPL_MODULE="source_googleanalytics_singer" ENV AIRBYTE_IMPL_PATH="GoogleAnalyticsSingerSource" -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.4 LABEL io.airbyte.name=airbyte/source-googleanalytics-singer WORKDIR /airbyte/integration_code diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml b/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml index 5acdf27946d0..43b826766789 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml @@ -13,14 +13,15 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "sample_files/configured_catalog.json" validate_output_from_all_streams: yes - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "sample_files/configured_catalog.json" - state_path: "sample_files/abnormal_state.json" - cursor_paths: - website_overview: ["report_start_date"] - traffic_sources: ["report_start_date"] - pages: ["report_start_date"] +# Commented out due to the state structure is not supported by the SAT +# incremental: +# - config_path: "secrets/config.json" +# configured_catalog_path: "sample_files/configured_catalog.json" +# state_path: "sample_files/abnormal_state.json" +# cursor_paths: +# website_overview: ["report_start_date"] +# traffic_sources: ["report_start_date"] +# pages: ["report_start_date"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json index 397b5b03d41f..c9d1a02acd5a 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json @@ -3,4 +3,4 @@ "view_id": "222222222", "start_date": "2020-02-13T00:00:00Z", "custom_reports": "[{\"name\": \"users_per_day\", \"dimensions\": [\"ga:date\"], \"metrics\": [\"ga:users\", \"ga:newUsers\"]}, {\"name\": \"sessions_per_country_day\", \"dimensions\": [\"ga:date\", \"ga:country\"], \"metrics\": [\"ga:sessions\", \"ga:sessionsPerUser\", \"ga:avgSessionDuration\"]}]" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json b/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json index 28ce1ab4f190..b94062f8e2a1 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json @@ -1,5 +1,5 @@ { - "website_overview": {"report_start_date": "2050-05-01T00:00:00Z"}, - "traffic_sources": {"report_start_date": "2050-05-01T00:00:00Z"}, - "pages": {"report_start_date": "2050-05-01T00:00:00Z"} -} \ No newline at end of file + "website_overview": { "report_start_date": "2050-05-01T00:00:00Z" }, + "traffic_sources": { "report_start_date": "2050-05-01T00:00:00Z" }, + "pages": { "report_start_date": "2050-05-01T00:00:00Z" } +} From 668fd754df9e1716cde943447ca21d5a5a4fe414 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Tue, 15 Jun 2021 10:16:17 +0300 Subject: [PATCH 12/19] version bump --- .../39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../connectors/source-googleanalytics-singer/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json index 69023fb04c98..ffc80a647046 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1", "name": "Google Analytics", "dockerRepository": "airbyte/source-googleanalytics-singer", - "dockerImageTag": "0.2.4", + "dockerImageTag": "0.2.5", "documentationUrl": "https://hub.docker.com/r/airbyte/source-googleanalytics-singer", "icon": "google-analytics.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d4397ebef362..f51cbe1c7784 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -95,7 +95,7 @@ - sourceDefinitionId: 39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1 name: Google Analytics dockerRepository: airbyte/source-googleanalytics-singer - dockerImageTag: 0.2.4 + dockerImageTag: 0.2.5 documentationUrl: https://hub.docker.com/r/airbyte/source-googleanalytics-singer icon: google-analytics.svg - sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile b/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile index 48ba0055582c..a4371c49c760 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile @@ -8,7 +8,7 @@ ENV CODE_PATH="source_googleanalytics_singer" ENV AIRBYTE_IMPL_MODULE="source_googleanalytics_singer" ENV AIRBYTE_IMPL_PATH="GoogleAnalyticsSingerSource" -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-googleanalytics-singer WORKDIR /airbyte/integration_code From 13f6345436c594c17ea3315e19419258501150a8 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Tue, 15 Jun 2021 11:43:29 +0300 Subject: [PATCH 13/19] build file fix --- .../connectors/source-googleanalytics-singer/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle b/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle index ba2e138288b1..a7f77917049c 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle @@ -13,7 +13,7 @@ dependencies { implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) } -} + // TODO(sherifnada) re-enable tests. Currently they're disabled because they hog up so much bandwidth that they prevent local iteration at times. // airbyteStandardSourceTestFile { From 43a2be5a860f617ac6d26b5fae07bf9d348d2ff9 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Tue, 15 Jun 2021 12:35:06 +0300 Subject: [PATCH 14/19] removed unrelated files --- .../destination/s3/S3FormatConfigs.java | 57 ------ .../destination/s3/S3OutputFormatter.java | 52 ----- .../s3/S3OutputFormatterFactory.java | 43 ---- .../S3OutputFormatterProductionFactory.java | 49 ----- .../s3/csv/S3CsvOutputFormatter.java | 192 ------------------ .../s3/S3CsvDestinationAcceptanceTest.java | 130 ------------ .../s3/csv/S3CsvOutputFormatterTest.java | 59 ------ .../acceptance-test-config.yml | 2 +- docs/troubleshooting/on-deploy.md | 69 ------- docs/troubleshooting/running-sync.md | 5 +- 10 files changed, 2 insertions(+), 656 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java delete mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatter.java delete mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterFactory.java delete mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterProductionFactory.java delete mode 100644 airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatter.java delete mode 100644 airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java delete mode 100644 airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatterTest.java delete mode 100644 docs/troubleshooting/on-deploy.md diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java deleted file mode 100644 index acc4de70d297..000000000000 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.integrations.destination.s3; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; - -import io.airbyte.integrations.destination.s3.parquet.S3ParquetFormatConfig; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class S3FormatConfigs { - - protected static final Logger LOGGER = LoggerFactory.getLogger(S3FormatConfigs.class); - - public static S3FormatConfig getS3FormatConfig(JsonNode config) { - JsonNode formatConfig = config.get("format"); - LOGGER.info("S3 format config: {}", formatConfig.toString()); - S3Format formatType = S3Format.valueOf(formatConfig.get("format_type").asText().toUpperCase()); - - switch (formatType) { - case CSV -> { - return new S3CsvFormatConfig(formatConfig); - } - case PARQUET -> { - return new S3ParquetFormatConfig(formatConfig); - } - default -> { - throw new RuntimeException("Unexpected output format: " + Jsons.serialize(config)); - } - } - } - -} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatter.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatter.java deleted file mode 100644 index 4be7b6f77b4e..000000000000 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatter.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.integrations.destination.s3; - -import io.airbyte.protocol.models.AirbyteRecordMessage; -import java.io.IOException; -import java.util.UUID; - -/** - * {@link S3OutputFormatter} is responsible for writing Airbyte stream data to an S3 location in a - * specific format. - */ -public interface S3OutputFormatter { - - /** - * Prepare an S3 writer for the stream. - */ - void initialize() throws IOException; - - /** - * Write an Airbyte record message to an S3 object. - */ - void write(UUID id, AirbyteRecordMessage recordMessage) throws IOException; - - /** - * Close the S3 writer for the stream. - */ - void close(boolean hasFailed) throws IOException; - -} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterFactory.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterFactory.java deleted file mode 100644 index f08dce169a24..000000000000 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterFactory.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.integrations.destination.s3; - -import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import java.io.IOException; -import java.sql.Timestamp; - -/** - * Create different {@link S3OutputFormatter} based on {@link S3DestinationConfig}. - */ -public interface S3OutputFormatterFactory { - - S3OutputFormatter create(S3DestinationConfig config, - AmazonS3 s3Client, - ConfiguredAirbyteStream configuredStream, - Timestamp uploadTimestamp) - throws IOException; - -} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterProductionFactory.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterProductionFactory.java deleted file mode 100644 index 582f3cfd8ea9..000000000000 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3OutputFormatterProductionFactory.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.integrations.destination.s3; - -import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.integrations.destination.s3.csv.S3CsvOutputFormatter; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import java.io.IOException; -import java.sql.Timestamp; - -public class S3OutputFormatterProductionFactory implements S3OutputFormatterFactory { - - @Override - public S3OutputFormatter create(S3DestinationConfig config, - AmazonS3 s3Client, - ConfiguredAirbyteStream configuredStream, - Timestamp uploadTimestamp) - throws IOException { - S3Format format = config.getFormatConfig().getFormat(); - if (format == S3Format.CSV) { - return new S3CsvOutputFormatter(config, s3Client, configuredStream, uploadTimestamp); - } - - throw new RuntimeException("Unexpected S3 destination format: " + format); - } - -} diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatter.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatter.java deleted file mode 100644 index d7f7fb31be91..000000000000 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatter.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.integrations.destination.s3.csv; - -import static io.airbyte.integrations.destination.s3.S3DestinationConstants.YYYY_MM_DD_FORMAT; - -import alex.mojaki.s3upload.MultiPartOutputStream; -import alex.mojaki.s3upload.StreamTransferManager; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; -import com.amazonaws.services.s3.model.DeleteObjectsResult; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.integrations.destination.ExtendedNameTransformer; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConstants; -import io.airbyte.integrations.destination.s3.S3OutputFormatter; -import io.airbyte.protocol.models.AirbyteRecordMessage; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.DestinationSyncMode; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.Timestamp; -import java.util.LinkedList; -import java.util.List; -import java.util.UUID; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.QuoteMode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class S3CsvOutputFormatter implements S3OutputFormatter { - - private static final Logger LOGGER = LoggerFactory.getLogger(S3CsvOutputFormatter.class); - private static final ExtendedNameTransformer NAME_TRANSFORMER = new ExtendedNameTransformer(); - - private final S3DestinationConfig config; - private final S3CsvFormatConfig formatConfig; - private final AmazonS3 s3Client; - private final AirbyteStream stream; - private final DestinationSyncMode syncMode; - private final CsvSheetGenerator csvSheetGenerator; - private final String outputPrefix; - private final StreamTransferManager uploadManager; - private final MultiPartOutputStream outputStream; - private final CSVPrinter csvPrinter; - - public S3CsvOutputFormatter(S3DestinationConfig config, - AmazonS3 s3Client, - ConfiguredAirbyteStream configuredStream, - Timestamp uploadTimestamp) - throws IOException { - this.config = config; - this.formatConfig = (S3CsvFormatConfig) config.getFormatConfig(); - this.s3Client = s3Client; - this.stream = configuredStream.getStream(); - this.syncMode = configuredStream.getDestinationSyncMode(); - this.csvSheetGenerator = CsvSheetGenerator.Factory.create(configuredStream.getStream().getJsonSchema(), formatConfig); - - // prefix: // - // filename: -.csv - // full path: // - this.outputPrefix = getOutputPrefix(config.getBucketPath(), stream); - String outputFilename = getOutputFilename(uploadTimestamp); - String objectKey = String.join("/", outputPrefix, outputFilename); - - LOGGER.info("Full S3 path for stream '{}': {}/{}", stream.getName(), config.getBucketName(), - objectKey); - - // The stream transfer manager lets us greedily stream into S3. The native AWS SDK does not - // have support for streaming multipart uploads. The alternative is first writing the entire - // output to disk before loading into S3. This is not feasible with large input. - // Data is chunked into parts during the upload. A part is sent off to a queue to be uploaded - // once it has reached it's configured part size. - // See {@link S3DestinationConstants} for memory usage calculation. - this.uploadManager = new StreamTransferManager(config.getBucketName(), objectKey, s3Client) - .numStreams(S3DestinationConstants.DEFAULT_NUM_STREAMS) - .queueCapacity(S3DestinationConstants.DEFAULT_QUEUE_CAPACITY) - .numUploadThreads(S3DestinationConstants.DEFAULT_UPLOAD_THREADS) - .partSize(S3DestinationConstants.DEFAULT_PART_SIZE_MD); - // We only need one output stream as we only have one input stream. This is reasonably performant. - this.outputStream = uploadManager.getMultiPartOutputStreams().get(0); - this.csvPrinter = new CSVPrinter(new PrintWriter(outputStream, true, StandardCharsets.UTF_8), - CSVFormat.DEFAULT.withQuoteMode(QuoteMode.ALL) - .withHeader(csvSheetGenerator.getHeaderRow().toArray(new String[0]))); - } - - static String getOutputPrefix(String bucketPath, AirbyteStream stream) { - return getOutputPrefix(bucketPath, stream.getNamespace(), stream.getName()); - } - - @VisibleForTesting - public static String getOutputPrefix(String bucketPath, String namespace, String streamName) { - List paths = new LinkedList<>(); - - if (bucketPath != null) { - paths.add(bucketPath); - } - if (namespace != null) { - paths.add(NAME_TRANSFORMER.convertStreamName(namespace)); - } - paths.add(NAME_TRANSFORMER.convertStreamName(streamName)); - - return String.join("/", paths); - } - - static String getOutputFilename(Timestamp timestamp) { - return String.format("%s_%d_0.csv", YYYY_MM_DD_FORMAT.format(timestamp), timestamp.getTime()); - } - - /** - *
  • 1. Create bucket if necessary.
  • - *
  • 2. Under OVERWRITE mode, delete all objects with the output prefix.
  • - */ - @Override - public void initialize() { - String bucket = config.getBucketName(); - if (!s3Client.doesBucketExistV2(bucket)) { - LOGGER.info("Bucket {} does not exist; creating...", bucket); - s3Client.createBucket(bucket); - LOGGER.info("Bucket {} has been created.", bucket); - } - - if (syncMode == DestinationSyncMode.OVERWRITE) { - LOGGER.info("Overwrite mode"); - List keysToDelete = new LinkedList<>(); - List objects = s3Client.listObjects(bucket, outputPrefix) - .getObjectSummaries(); - for (S3ObjectSummary object : objects) { - keysToDelete.add(new KeyVersion(object.getKey())); - } - - if (keysToDelete.size() > 0) { - LOGGER.info("Purging non-empty output path for stream '{}' under OVERWRITE mode...", - stream.getName()); - DeleteObjectsResult result = s3Client - .deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keysToDelete)); - LOGGER.info("Deleted {} file(s) for stream '{}'.", result.getDeletedObjects().size(), - stream.getName()); - } - } - } - - @Override - public void write(UUID id, AirbyteRecordMessage recordMessage) throws IOException { - csvPrinter.printRecord(csvSheetGenerator.getDataRow(id, recordMessage)); - } - - @Override - public void close(boolean hasFailed) throws IOException { - if (hasFailed) { - LOGGER.warn("Failure detected. Aborting upload of stream '{}'...", stream.getName()); - csvPrinter.close(); - outputStream.close(); - uploadManager.abort(); - LOGGER.warn("Upload of stream '{}' aborted.", stream.getName()); - } else { - LOGGER.info("Uploading remaining data for stream '{}'.", stream.getName()); - csvPrinter.close(); - outputStream.close(); - uploadManager.complete(); - LOGGER.info("Upload completed for stream '{}'.", stream.getName()); - } - } - -} diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java deleted file mode 100644 index 2bb9cf5a38a2..000000000000 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.integrations.destination.s3; - - -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.Reader; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.stream.StreamSupport; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVRecord; -import org.apache.commons.csv.QuoteMode; - -public class S3CsvDestinationAcceptanceTest extends S3DestinationAcceptanceTest { - - public S3CsvDestinationAcceptanceTest() { - super(S3Format.CSV); - } - - @Override - protected JsonNode getFormatConfig() { - return Jsons.deserialize("{\n" - + " \"format_type\": \"CSV\",\n" - + " \"flattening\": \"Root level flattening\"\n" - + "}"); - } - - /** - * Convert json_schema to a map from field name to field types. - */ - private static Map getFieldTypes(JsonNode streamSchema) { - Map fieldTypes = new HashMap<>(); - JsonNode fieldDefinitions = streamSchema.get("properties"); - Iterator> iterator = fieldDefinitions.fields(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - fieldTypes.put(entry.getKey(), entry.getValue().get("type").asText()); - } - return fieldTypes; - } - - - private static JsonNode getJsonNode(Map input, Map fieldTypes) { - ObjectNode json = MAPPER.createObjectNode(); - if (input.containsKey(JavaBaseConstants.COLUMN_NAME_DATA)) { - return Jsons.deserialize(input.get(JavaBaseConstants.COLUMN_NAME_DATA)); - } - - for (Map.Entry entry : input.entrySet()) { - String key = entry.getKey(); - if (key.equals(JavaBaseConstants.COLUMN_NAME_AB_ID) || key - .equals(JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) { - continue; - } - String value = entry.getValue(); - if (value == null || value.equals("")) { - continue; - } - String type = fieldTypes.get(key); - switch (type) { - case "boolean" -> json.put(key, Boolean.valueOf(value)); - case "integer" -> json.put(key, Integer.valueOf(value)); - case "number" -> json.put(key, Double.valueOf(value)); - default -> json.put(key, value); - } - } - return json; - } - - - @Override - protected List retrieveRecords(TestDestinationEnv testEnv, - String streamName, - String namespace, - JsonNode streamSchema) - throws IOException { - List objectSummaries = getAllSyncedObjects(streamName, namespace); - Map fieldTypes = getFieldTypes(streamSchema); - List jsonRecords = new LinkedList<>(); - - for (S3ObjectSummary objectSummary : objectSummaries) { - S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); - Reader in = new InputStreamReader(object.getObjectContent(), StandardCharsets.UTF_8); - Iterable records = CSVFormat.DEFAULT - .withQuoteMode(QuoteMode.NON_NUMERIC) - .withFirstRecordAsHeader() - .parse(in); - StreamSupport.stream(records.spliterator(), false) - .forEach(r -> jsonRecords.add(getJsonNode(r.toMap(), fieldTypes))); - } - - return jsonRecords; - } - -} diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatterTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatterTest.java deleted file mode 100644 index c2d6972c3233..000000000000 --- a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvOutputFormatterTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2020 Airbyte - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package io.airbyte.integrations.destination.s3.csv; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.protocol.models.AirbyteStream; -import java.sql.Timestamp; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -@DisplayName("S3CsvOutputFormatter") -class S3CsvOutputFormatterTest { - - @Test - @DisplayName("getOutputPrefix") - public void testGetOutputPrefix() { - // No namespace - assertEquals("bucket_path/stream_name", S3CsvOutputFormatter - .getOutputPrefix("bucket_path", new AirbyteStream().withName("stream_name"))); - - // With namespace - assertEquals("bucket_path/namespace/stream_name", S3CsvOutputFormatter - .getOutputPrefix("bucket_path", - new AirbyteStream().withNamespace("namespace").withName("stream_name"))); - } - - @Test - @DisplayName("getOutputFilename") - public void testGetOutputFilename() { - Timestamp timestamp = new Timestamp(1471461319000L); - assertEquals( - "2016_08_17_1471461319000_0.csv", - S3CsvOutputFormatter.getOutputFilename(timestamp)); - } - -} diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml b/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml index 43b826766789..0f077cd3b443 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml @@ -17,7 +17,7 @@ tests: # incremental: # - config_path: "secrets/config.json" # configured_catalog_path: "sample_files/configured_catalog.json" -# state_path: "sample_files/abnormal_state.json" +# future_state_path: "sample_files/abnormal_state.json" # cursor_paths: # website_overview: ["report_start_date"] # traffic_sources: ["report_start_date"] diff --git a/docs/troubleshooting/on-deploy.md b/docs/troubleshooting/on-deploy.md deleted file mode 100644 index 10783e8e3163..000000000000 --- a/docs/troubleshooting/on-deploy.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -description: Common issues and their workarounds related when trying launch Airbyte ---- - -# On deployment - -## Stuck in onboarding, can’t skip or do anything -To full reset Airbyte, you need to also delete the docker volumes associated with Airbyte. This is where data is stored. -Assuming that you are running Airbyte by running `docker-compose up`, then what you need to do is: -* Turn off Airbyte completely: `docker-compose down -v` -* Turn Airbyte back on: `docker-compose up` - -that should handle you getting reset to the beginning. -I would be curious if we can see the logs associated with the failure you are seeing. I would say if after you reset you run into it again we can debug that. - - -### I have run `docker-compose up` and can not access the interface - -- If you see a blank screen and not a loading icon: - -Check your web browser version; Some old versions of web browsers doesn't support our current Front-end stack. - -- If you see a loading icon or the message `Cannot reach the server` persist: - -Check if all Airbyte containers are running, executing: `docker ps` - -```text -CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES -f45f3cfe1e16 airbyte/scheduler:1.11.1-alpha "/bin/bash -c './wai…" 2 hours ago Up 2 hours airbyte-scheduler -f02fc709b130 airbyte/server:1.11.1-alpha "/bin/bash -c './wai…" 2 hours ago Up 2 hours 8000/tcp, [...] :::8001->8001/tcp airbyte-server -153b2b322870 airbyte/webapp:1.11.1-alpha "/docker-entrypoint.…" 2 hours ago Up 2 hours :::8000->80/tcp airbyte-webapp -b88d94652268 airbyte/db:1.11.1-alpha "docker-entrypoint.s…" 2 hours ago Up 2 hours 5432/tcp airbyte-db -0573681a10e0 temporalio/auto-setup:1.7.0 "/entrypoint.sh /bin…" 2 hours ago Up 2 hours 6933-6935/tcp, [...] airbyte-temporal -``` -You must see 5 containers running. If you are not seeing execute the following steps: -* `docker-compose down -v` -* `docker-compose up` -Keep in mind the commands above will delete ALL containers, volumes and data created by Airbyte. - We do not recommend this is you already deploy and have connection created. - -First, let's check the server logs by running `docker logs airbyte-server | grep ERROR`.
    -If this command returns any output, please run `docker logs airbyte-server > airbyte-server.log`.
    -This command will create a file in the current directory. We advise you to send a message on our #issues on Slack channel - -If you don't have any server errors let's check the scheduler, `docker logs airbyte-scheduler | grep ERROR`.
    -If this command returns any output, please run `docker logs airbyte-scheduler > airbyte-scheduler.log`.
    -This command will create a file in the current directory. We advise you to send a message on our #issues on Slack channel - -If there is no error printed in both cases, we recommend running: `docker restart airbyte-server airbyte-scheduler`
    -Wait a few moments and try to access the interface again. - -### `docker.errors.DockerException`: Error while fetching server API version - -If you see the following error: - -```text -docker.errors.DockerException: Error while fetching server API -version: ('Connection aborted.', FileNotFoundError(2, 'No such file or -directory')) -``` - -It usually means that Docker isn't running on your machine \(and a running Docker daemon is required to run Airbyte\). An easy way to verify this is to run `docker ps`, which will show `Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?` if the Docker daemon is not running on your machine. - -This happens (sometimes) on Windows system when you first install `docker`. You need to restart your machine. - - -### Getting a weird error related to setting up the Airbyte server when running Docker Compose -- wondering if this is because I played around with Airbyte in a past version? - -If you are okay with losing your previous Airbyte configurations, you can run `docker-compose down -v` and that should fix things then `docker-compose up`. diff --git a/docs/troubleshooting/running-sync.md b/docs/troubleshooting/running-sync.md index 4610c6c4d36c..1dc4a40fcff1 100644 --- a/docs/troubleshooting/running-sync.md +++ b/docs/troubleshooting/running-sync.md @@ -1,4 +1,3 @@ - # On Running a Sync ## One of your sync jobs is failing @@ -6,7 +5,6 @@ Several things to check: * **Is Airbyte updated to your latest version?** You can see the latest version [here](https://github.com/airbytehq/airbyte/tags). If not, please upgrade to the latest one, [upgrading instructions are here](../operator-guides/upgrading-airbyte.md) - * **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-config/init/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). @@ -18,7 +16,6 @@ Our current version of incremental is [append](../understanding-airbyte/connecti If this is true, then, there are still several things to check: * **Is Airbyte updated to your latest version?** You can see the latest version [here](https://github.com/airbytehq/airbyte/tags). If not, please upgrade to the latest one, [upgrading instructions are here](../operator-guides/upgrading-airbyte.md) - * **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-config/init/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). @@ -32,4 +29,4 @@ Several things to check: * **Is Airbyte updated to your latest version?** You can see the latest version [here](https://github.com/airbytehq/airbyte/tags). If not, please upgrade to the latest one, [upgrading instructions are here](../operator-guides/upgrading-airbyte.md) * **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-config/init/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. -If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). +If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). \ No newline at end of file From 12b5225e462992c094e99487e4945eec875add31 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Tue, 15 Jun 2021 12:52:45 +0300 Subject: [PATCH 15/19] removed unrelated files --- docs/troubleshooting/upgrading.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 docs/troubleshooting/upgrading.md diff --git a/docs/troubleshooting/upgrading.md b/docs/troubleshooting/upgrading.md deleted file mode 100644 index e69de29bb2d1..000000000000 From 83c7e76fc8f42d361d51842bf580bc001ee2c3a3 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Tue, 15 Jun 2021 15:06:37 +0300 Subject: [PATCH 16/19] removed unrelated files --- .../s3/S3CsvDestinationAcceptanceTest.java | 129 ++++++++++++++++++ .../destination/s3/S3FormatConfigs.java | 56 ++++++++ 2 files changed, 185 insertions(+) create mode 100644 airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java create mode 100644 airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java new file mode 100644 index 000000000000..3ffb285974a4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java @@ -0,0 +1,129 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.JavaBaseConstants; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.StreamSupport; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; +import org.apache.commons.csv.QuoteMode; + +public class S3CsvDestinationAcceptanceTest extends S3DestinationAcceptanceTest { + + public S3CsvDestinationAcceptanceTest() { + super(S3Format.CSV); + } + + @Override + protected JsonNode getFormatConfig() { + return Jsons.deserialize("{\n" + + " \"format_type\": \"CSV\",\n" + + " \"flattening\": \"Root level flattening\"\n" + + "}"); + } + + /** + * Convert json_schema to a map from field name to field types. + */ + private static Map getFieldTypes(JsonNode streamSchema) { + Map fieldTypes = new HashMap<>(); + JsonNode fieldDefinitions = streamSchema.get("properties"); + Iterator> iterator = fieldDefinitions.fields(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + fieldTypes.put(entry.getKey(), entry.getValue().get("type").asText()); + } + return fieldTypes; + } + + private static JsonNode getJsonNode(Map input, Map fieldTypes) { + ObjectNode json = MAPPER.createObjectNode(); + + if (input.containsKey(JavaBaseConstants.COLUMN_NAME_DATA)) { + return Jsons.deserialize(input.get(JavaBaseConstants.COLUMN_NAME_DATA)); + } + + for (Map.Entry entry : input.entrySet()) { + String key = entry.getKey(); + if (key.equals(JavaBaseConstants.COLUMN_NAME_AB_ID) || key + .equals(JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) { + continue; + } + String value = entry.getValue(); + if (value == null || value.equals("")) { + continue; + } + String type = fieldTypes.get(key); + switch (type) { + case "boolean" -> json.put(key, Boolean.valueOf(value)); + case "integer" -> json.put(key, Integer.valueOf(value)); + case "number" -> json.put(key, Double.valueOf(value)); + default -> json.put(key, value); + } + } + return json; + } + + @Override + protected List retrieveRecords(TestDestinationEnv testEnv, + String streamName, + String namespace, + JsonNode streamSchema) + throws IOException { + List objectSummaries = getAllSyncedObjects(streamName, namespace); + + Map fieldTypes = getFieldTypes(streamSchema); + List jsonRecords = new LinkedList<>(); + + for (S3ObjectSummary objectSummary : objectSummaries) { + S3Object object = s3Client.getObject(objectSummary.getBucketName(), objectSummary.getKey()); + Reader in = new InputStreamReader(object.getObjectContent(), StandardCharsets.UTF_8); + Iterable records = CSVFormat.DEFAULT + .withQuoteMode(QuoteMode.NON_NUMERIC) + .withFirstRecordAsHeader() + .parse(in); + StreamSupport.stream(records.spliterator(), false) + .forEach(r -> jsonRecords.add(getJsonNode(r.toMap(), fieldTypes))); + } + + return jsonRecords; + } + +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java new file mode 100644 index 000000000000..73378994bd0a --- /dev/null +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java @@ -0,0 +1,56 @@ +/* + * MIT License + * + * Copyright (c) 2020 Airbyte + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.airbyte.integrations.destination.s3; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; +import io.airbyte.integrations.destination.s3.parquet.S3ParquetFormatConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class S3FormatConfigs { + + protected static final Logger LOGGER = LoggerFactory.getLogger(S3FormatConfigs.class); + + public static S3FormatConfig getS3FormatConfig(JsonNode config) { + JsonNode formatConfig = config.get("format"); + LOGGER.info("S3 format config: {}", formatConfig.toString()); + S3Format formatType = S3Format.valueOf(formatConfig.get("format_type").asText().toUpperCase()); + + switch (formatType) { + case CSV -> { + return new S3CsvFormatConfig(formatConfig); + } + case PARQUET -> { + return new S3ParquetFormatConfig(formatConfig); + } + default -> { + throw new RuntimeException("Unexpected output format: " + Jsons.serialize(config)); + } + } + } + +} \ No newline at end of file From 6095166902cd0cdbea3f71f0fee263a28b12ff16 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Tue, 15 Jun 2021 15:11:56 +0300 Subject: [PATCH 17/19] removed unrelated files --- .../destination/s3/S3CsvDestinationAcceptanceTest.java | 2 +- .../io/airbyte/integrations/destination/s3/S3FormatConfigs.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java index 3ffb285974a4..734ee29e07c0 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java @@ -126,4 +126,4 @@ protected List retrieveRecords(TestDestinationEnv testEnv, return jsonRecords; } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java index 73378994bd0a..063f29e8f9ee 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java @@ -53,4 +53,4 @@ public static S3FormatConfig getS3FormatConfig(JsonNode config) { } } -} \ No newline at end of file +} From 1de9852921a42fbcaf0f8a235b0bfbdc2fda0d6a Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Tue, 15 Jun 2021 15:19:07 +0300 Subject: [PATCH 18/19] removed unrelated files --- .../destination/s3/S3CsvDestinationAcceptanceTest.java | 0 .../io/airbyte/integrations/destination/s3/S3FormatConfigs.java | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename airbyte-integrations/connectors/destination-s3/src/{test => main}/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java (100%) rename airbyte-integrations/connectors/destination-s3/src/{test => main}/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java (100%) diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java similarity index 100% rename from airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java rename to airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java similarity index 100% rename from airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java rename to airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java From d1775f194971b8b8eb6969aa03fc26383d533354 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Tue, 15 Jun 2021 15:20:32 +0300 Subject: [PATCH 19/19] removed unrelated files --- .../destination/s3/S3CsvDestinationAcceptanceTest.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename airbyte-integrations/connectors/destination-s3/src/{main => test-integration}/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java (100%) diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java similarity index 100% rename from airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java rename to airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java