From 40c209f7e6654fdda584fcf8c81a872f51626bb6 Mon Sep 17 00:00:00 2001 From: Titas Skrebe Date: Tue, 1 Mar 2022 12:53:35 +0200 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=8E=89=20Source=20Chartmogul:=20Add?= =?UTF-8?q?=20CustomerCount=20stream?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/seed/source_definitions.yaml | 2 +- .../connectors/source-chartmogul/Dockerfile | 2 +- .../connectors/source-chartmogul/bootstrap.md | 1 + .../integration_tests/configured_catalog.json | 9 +++ .../integration_tests/invalid_config.json | 3 +- .../integration_tests/sample_config.json | 3 +- .../sample_files/configured_catalog.json | 9 +++ .../schemas/customer_count.json | 12 ++++ .../source_chartmogul/source.py | 59 ++++++++++++++++++- .../source_chartmogul/spec.json | 9 ++- .../unit_tests/test_source.py | 2 +- docs/integrations/sources/chartmogul.md | 2 + 12 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_count.json diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 738c638e3c21..3f8a163e9953 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -119,7 +119,7 @@ - name: Chartmogul sourceDefinitionId: b6604cbd-1b12-4c08-8767-e140d0fb0877 dockerRepository: airbyte/source-chartmogul - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/chartmogul icon: chartmogul.svg sourceType: api diff --git a/airbyte-integrations/connectors/source-chartmogul/Dockerfile b/airbyte-integrations/connectors/source-chartmogul/Dockerfile index 35cef5c73022..bf9eff300902 100644 --- a/airbyte-integrations/connectors/source-chartmogul/Dockerfile +++ b/airbyte-integrations/connectors/source-chartmogul/Dockerfile @@ -34,5 +34,5 @@ COPY source_chartmogul ./source_chartmogul ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-chartmogul diff --git a/airbyte-integrations/connectors/source-chartmogul/bootstrap.md b/airbyte-integrations/connectors/source-chartmogul/bootstrap.md index d6fee29243da..c7dfde476b2a 100644 --- a/airbyte-integrations/connectors/source-chartmogul/bootstrap.md +++ b/airbyte-integrations/connectors/source-chartmogul/bootstrap.md @@ -5,6 +5,7 @@ Chartmogul is an online subscription analytics platform. It retrieves data from Connector currently implements following full refresh streams: * [Customers](https://dev.chartmogul.com/reference/list-customers) +* [CustomerCount] (https://dev.chartmogul.com/reference/retrieve-customer-count) * [Activities](https://dev.chartmogul.com/reference/list-activities) `start_date` config is used for retrieving `Activies`. `Customers` stream does not use this config. Even if it was possible to filter by `start_date`, it would cause issues when modeling data. That is because activies after `start_date` can be triggered by customers who were created way before that. diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json index 3b788b27041d..9ba89f7766d2 100644 --- a/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json @@ -17,6 +17,15 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customer_count", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json index dc521ade7acf..2460b6de2218 100644 --- a/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json @@ -1,3 +1,4 @@ { - "api_key": "" + "api_key": "", + "interval": "day" } diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json index c09fca36f467..7d1a7b8f198f 100644 --- a/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json @@ -1,4 +1,5 @@ { "api_key": "", - "start_date": "2022-01-05T12:09:00Z" + "start_date": "2022-01-05T12:09:00Z", + "interval": "day" } diff --git a/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json index 3b788b27041d..9ba89f7766d2 100644 --- a/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json @@ -17,6 +17,15 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customer_count", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_count.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_count.json new file mode 100644 index 000000000000..771eb81f0f9a --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_count.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "date": { + "type": ["string"] + }, + "customers": { + "type": ["integer"] + } + } + } diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py index 6a3a8fb675fe..490bb0cff325 100644 --- a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py @@ -5,13 +5,15 @@ from abc import ABC from base64 import b64encode from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from urllib.parse import urljoin +from datetime import datetime import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator - +from airbyte_cdk.sources.streams.http.exceptions import RequestBodyException # Basic full refresh stream class ChartmogulStream(HttpStream, ABC): @@ -70,6 +72,55 @@ def path(self, **kwargs) -> str: return "v1/activities" +class CustomerCount(ChartmogulStream): + primary_key = "date" + + # customer-count API requires start date. + default_start_date = "2020-01-01" + + def __init__(self, start_date: str, interval: str, **kwargs): + super().__init__(**kwargs) + self.start_date = start_date or self.default_start_date + self.end_date = datetime.now().strftime("%Y-%m-%d") + self.interval = interval + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def request_body_data( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Optional[Mapping]: + return { + "start-date": self.start_date, + "end-date": self.end_date, + "interval": self.interval, + } + + def _create_prepared_request( + self, path: str, headers: Mapping = None, params: Mapping = None, json: Any = None, data: Any = None + ) -> requests.PreparedRequest: + """ + Override to make possible sending http body with GET request. + """ + args = {"method": self.http_method, "url": urljoin(self.url_base, path), "headers": headers, "params": params} + if json and data: + raise RequestBodyException( + "At the same time only one of the 'request_body_data' and 'request_body_json' functions can return data" + ) + elif json: + args["json"] = json + elif data: + args["data"] = data + + return self._session.prepare_request(requests.Request(**args)) + + def path(self, **kwargs) -> str: + return "v1/metrics/customer-count" + + class HttpBasicAuthenticator(TokenAuthenticator): def __init__(self, token: str, auth_method: str = "Basic", **kwargs): auth_string = f"{token}:".encode("utf8") @@ -91,4 +142,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = HttpBasicAuthenticator(config["api_key"], auth_method="Basic") - return [Customers(authenticator=auth), Activities(authenticator=auth, start_date=config.get("start_date"))] + return [ + Customers(authenticator=auth), + CustomerCount(authenticator=auth, start_date=config.get("start_date"), interval=config.get("interval")), + Activities(authenticator=auth, start_date=config.get("start_date")), + ] diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json index d6f16407e70c..835b0e5bb223 100644 --- a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Chartmogul Spec", "type": "object", - "required": ["api_key"], + "required": ["api_key", "interval"], "additionalProperties": false, "properties": { "api_key": { @@ -19,6 +19,13 @@ "description": "UTC date and time in the format 2017-01-25T00:00:00Z. When feasible, any data before this date will not be replicated.", "examples": ["2017-01-25T00:00:00Z"], "order": 1 + }, + "interval": { + "type": "string", + "description": "Metrics APIs uses intervals to cluster data. Default interval is \"month\"", + "enum": ["day", "week", "month", "quarter"], + "default": "month", + "order": 2 } } } diff --git a/airbyte-integrations/connectors/source-chartmogul/unit_tests/test_source.py b/airbyte-integrations/connectors/source-chartmogul/unit_tests/test_source.py index 32b777492434..7eaa76a7c268 100644 --- a/airbyte-integrations/connectors/source-chartmogul/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-chartmogul/unit_tests/test_source.py @@ -26,5 +26,5 @@ def test_streams(mocker): source = SourceChartmogul() config_mock = MagicMock() streams = source.streams(config_mock) - expected_streams_number = 2 + expected_streams_number = 3 assert len(streams) == expected_streams_number diff --git a/docs/integrations/sources/chartmogul.md b/docs/integrations/sources/chartmogul.md index e8d19c2ddda2..40935d02890b 100644 --- a/docs/integrations/sources/chartmogul.md +++ b/docs/integrations/sources/chartmogul.md @@ -15,6 +15,7 @@ If `start_date` is set, it will only apply to `Activities` stream. `Customers`' This Source is capable of syncing the following streams: * [Customers](https://dev.chartmogul.com/reference/list-customers) +* [CustomerCount] (https://dev.chartmogul.com/reference/retrieve-customer-count) * [Activities](https://dev.chartmogul.com/reference/list-activities) ### Features @@ -44,4 +45,5 @@ Please read [How to find your API key](https://dev.chartmogul.com/docs/authentic | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.1 | 2022-03-02 | [9381](https://github.com/airbytehq/airbyte/pull/xxx) | Add new stream: customer-count | | 0.1.0 | 2022-01-10 | [9381](https://github.com/airbytehq/airbyte/pull/9381) | New Source: Chartmogul | From 0b388c0cd78c14533e358abd17909b5d17022021 Mon Sep 17 00:00:00 2001 From: Titas Skrebe Date: Tue, 1 Mar 2022 14:35:25 +0200 Subject: [PATCH 2/6] Update description --- .../connectors/source-chartmogul/source_chartmogul/spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json index 835b0e5bb223..d5d869f0e495 100644 --- a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json @@ -22,7 +22,7 @@ }, "interval": { "type": "string", - "description": "Metrics APIs uses intervals to cluster data. Default interval is \"month\"", + "description": "Some APIs such as Metrics requires intervals to cluster data.", "enum": ["day", "week", "month", "quarter"], "default": "month", "order": 2 From ca4828107041f96f8c4b7c0bbd4fc46ec18cd734 Mon Sep 17 00:00:00 2001 From: Titas Skrebe Date: Wed, 2 Mar 2022 09:43:16 +0200 Subject: [PATCH 3/6] address comments --- .../source-chartmogul/integration_tests/invalid_config.json | 1 + .../connectors/source-chartmogul/source_chartmogul/source.py | 5 +---- .../connectors/source-chartmogul/source_chartmogul/spec.json | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json index 2460b6de2218..da1c2443aeb4 100644 --- a/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json @@ -1,4 +1,5 @@ { "api_key": "", + "start_date": "2017-01-25T00:00:00Z", "interval": "day" } diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py index 490bb0cff325..d214e531e43f 100644 --- a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py @@ -75,12 +75,9 @@ def path(self, **kwargs) -> str: class CustomerCount(ChartmogulStream): primary_key = "date" - # customer-count API requires start date. - default_start_date = "2020-01-01" - def __init__(self, start_date: str, interval: str, **kwargs): super().__init__(**kwargs) - self.start_date = start_date or self.default_start_date + self.start_date = start_date self.end_date = datetime.now().strftime("%Y-%m-%d") self.interval = interval diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json index d5d869f0e495..ff3d92dec022 100644 --- a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Chartmogul Spec", "type": "object", - "required": ["api_key", "interval"], + "required": ["api_key", "start_date", "interval"], "additionalProperties": false, "properties": { "api_key": { @@ -22,7 +22,7 @@ }, "interval": { "type": "string", - "description": "Some APIs such as Metrics requires intervals to cluster data.", + "description": "Some APIs such as Metrics require intervals to cluster data.", "enum": ["day", "week", "month", "quarter"], "default": "month", "order": 2 From 68406330e03bbb5e48a640f37bc6716211669147 Mon Sep 17 00:00:00 2001 From: Titas Skrebe Date: Wed, 2 Mar 2022 10:50:06 +0200 Subject: [PATCH 4/6] update changelog --- docs/integrations/sources/chartmogul.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/chartmogul.md b/docs/integrations/sources/chartmogul.md index 40935d02890b..74774d120352 100644 --- a/docs/integrations/sources/chartmogul.md +++ b/docs/integrations/sources/chartmogul.md @@ -45,5 +45,5 @@ Please read [How to find your API key](https://dev.chartmogul.com/docs/authentic | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.1 | 2022-03-02 | [9381](https://github.com/airbytehq/airbyte/pull/xxx) | Add new stream: customer-count | +| 0.1.1 | 2022-03-02 | [10756](https://github.com/airbytehq/airbyte/pull/10756) | Add new stream: customer-count | | 0.1.0 | 2022-01-10 | [9381](https://github.com/airbytehq/airbyte/pull/9381) | New Source: Chartmogul | From 216d99b6c19ea6c2c40872ec9d6ca9aaa45fbf3a Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 2 Mar 2022 18:22:56 -0300 Subject: [PATCH 5/6] format source file --- .../connectors/source-chartmogul/source_chartmogul/source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py index d214e531e43f..7f83d9e1b168 100644 --- a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/source.py @@ -4,9 +4,9 @@ from abc import ABC from base64 import b64encode +from datetime import datetime from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple from urllib.parse import urljoin -from datetime import datetime import requests from airbyte_cdk.sources import AbstractSource @@ -15,6 +15,7 @@ from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from airbyte_cdk.sources.streams.http.exceptions import RequestBodyException + # Basic full refresh stream class ChartmogulStream(HttpStream, ABC): url_base = "https://api.chartmogul.com" From 2bc5c06450a87a9a5aa93ae3883ca11042f1a09c Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Wed, 2 Mar 2022 19:00:40 -0300 Subject: [PATCH 6/6] run seed file --- .../src/main/resources/seed/source_specs.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index f0800c2d89d7..0984339c24bd 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1000,7 +1000,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-chartmogul:0.1.0" +- dockerImage: "airbyte/source-chartmogul:0.1.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/chartmogul" connectionSpecification: @@ -1009,6 +1009,8 @@ type: "object" required: - "api_key" + - "start_date" + - "interval" additionalProperties: false properties: api_key: @@ -1024,6 +1026,17 @@ examples: - "2017-01-25T00:00:00Z" order: 1 + interval: + type: "string" + description: "Some APIs such as Metrics require intervals to cluster data." + enum: + - "day" + - "week" + - "month" + - "quarter" + default: "month" + order: 2 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: []