From c53c04df4a5b5785f99a7869936498caeb02253c Mon Sep 17 00:00:00 2001 From: coeurdestenebres Date: Tue, 14 Sep 2021 10:32:16 -0500 Subject: [PATCH 1/5] Add support for self-host PostHog instances --- .../source-posthog/source_posthog/source.py | 28 +++++++++++-------- .../source-posthog/source_posthog/spec.json | 12 +++++++- .../source-posthog/source_posthog/streams.py | 14 ++++++++-- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/source.py b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py index 60e925528225..1c29c63e818c 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/source.py +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py @@ -46,13 +46,17 @@ Trends, ) +DEFAULT_BASE_URL = "https://app.posthog.com" + class SourcePosthog(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: _ = pendulum.parse(config["start_date"]) authenticator = TokenAuthenticator(token=config["api_key"]) - stream = PingMe(authenticator=authenticator) + base_url = config.get("base_url", DEFAULT_BASE_URL) + + stream = PingMe(authenticator=authenticator, base_url=base_url) records = stream.read_records(sync_mode=SyncMode.full_refresh) _ = next(records) return True, None @@ -69,15 +73,17 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: This stream was requested to be removed due to this reason. """ authenticator = TokenAuthenticator(token=config["api_key"]) + base_url = config.get("base_url", DEFAULT_BASE_URL) + return [ - Annotations(authenticator=authenticator, start_date=config["start_date"]), - Cohorts(authenticator=authenticator), - Events(authenticator=authenticator, start_date=config["start_date"]), - EventsSessions(authenticator=authenticator), - FeatureFlags(authenticator=authenticator), - Insights(authenticator=authenticator), - InsightsPath(authenticator=authenticator), - InsightsSessions(authenticator=authenticator), - Persons(authenticator=authenticator), - Trends(authenticator=authenticator), + Annotations(authenticator=authenticator, start_date=config["start_date"], base_url=base_url), + Cohorts(authenticator=authenticator, base_url=base_url), + Events(authenticator=authenticator, start_date=config["start_date"], base_url=base_url), + EventsSessions(authenticator=authenticator, base_url=base_url), + FeatureFlags(authenticator=authenticator, base_url=base_url), + Insights(authenticator=authenticator, base_url=base_url), + InsightsPath(authenticator=authenticator, base_url=base_url), + InsightsSessions(authenticator=authenticator, base_url=base_url), + Persons(authenticator=authenticator, base_url=base_url), + Trends(authenticator=authenticator, base_url=base_url), ] diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json b/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json index 59e6afb5cb1f..6fed07820adb 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json @@ -12,12 +12,22 @@ "type": "string", "description": "The date from which you'd like to replicate the data", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": "2021-01-01T00:00:00.000000Z" + "examples": [ + "2021-01-01T00:00:00.000000Z" + ] }, "api_key": { "type": "string", "airbyte_secret": true, "description": "API Key. See the docs for information on how to generate this key." + }, + "base_url": { + "type": "string", + "default": "https://app.posthog.com", + "description": "Base PostHog url. Defaults to PostHog Cloud (https://app.posthog.com).", + "examples": [ + "https://posthog.example.com" + ] } } } diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py b/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py index aa783f03823f..091ba64d0926 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py @@ -34,10 +34,18 @@ class PosthogStream(HttpStream, ABC): - url_base = "https://app.posthog.com/api/" + # url_base = "https://app.posthog.com/api/" primary_key = "id" data_field = "results" + def __init__(self, base_url: str, **kwargs): + super().__init__(**kwargs) + self._url_base = f"{base_url}/api/" + + @property + def url_base(self) -> str: + return self._url_base + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: resp_json = response.json() if resp_json.get("next"): @@ -76,8 +84,8 @@ class IncrementalPosthogStream(PosthogStream, ABC): state_checkpoint_interval = math.inf - def __init__(self, start_date: str, **kwargs): - super().__init__(**kwargs) + def __init__(self, base_url: str, start_date: str, **kwargs): + super().__init__(base_url=base_url, **kwargs) self._start_date = start_date self._initial_state = None # we need to keep it here because next_page_token doesn't accept state argument From 2796fd403bcfacb082a47b81b5cd6fb4eaca09b2 Mon Sep 17 00:00:00 2001 From: coeurdestenebres Date: Tue, 14 Sep 2021 10:37:38 -0500 Subject: [PATCH 2/5] Remove comment --- .../connectors/source-posthog/source_posthog/streams.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py b/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py index 091ba64d0926..52f8303a161e 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py @@ -34,7 +34,6 @@ class PosthogStream(HttpStream, ABC): - # url_base = "https://app.posthog.com/api/" primary_key = "id" data_field = "results" From 8accc9c4a1588e3461a90e4473055c2e92e2ed16 Mon Sep 17 00:00:00 2001 From: coeurdestenebres Date: Tue, 14 Sep 2021 10:42:42 -0500 Subject: [PATCH 3/5] Bump connector version --- .../af6d50ee-dddf-4126-a8ee-7faee990774f.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-integrations/connectors/source-posthog/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/af6d50ee-dddf-4126-a8ee-7faee990774f.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/af6d50ee-dddf-4126-a8ee-7faee990774f.json index da4acab7745e..8494bbc934f9 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/af6d50ee-dddf-4126-a8ee-7faee990774f.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/af6d50ee-dddf-4126-a8ee-7faee990774f.json @@ -2,6 +2,6 @@ "sourceDefinitionId": "af6d50ee-dddf-4126-a8ee-7faee990774f", "name": "PostHog", "dockerRepository": "airbyte/source-posthog", - "dockerImageTag": "0.1.3", + "dockerImageTag": "0.1.4", "documentationUrl": "https://docs.airbyte.io/integrations/sources/posthog" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b4a10c75dbe5..e46d21183f32 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -67,7 +67,7 @@ - sourceDefinitionId: af6d50ee-dddf-4126-a8ee-7faee990774f name: PostHog dockerRepository: airbyte/source-posthog - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/posthog - sourceDefinitionId: cd42861b-01fc-4658-a8ab-5d11d0510f01 name: Recurly diff --git a/airbyte-integrations/connectors/source-posthog/Dockerfile b/airbyte-integrations/connectors/source-posthog/Dockerfile index d610222cd6aa..bf17d72d4cdb 100644 --- a/airbyte-integrations/connectors/source-posthog/Dockerfile +++ b/airbyte-integrations/connectors/source-posthog/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-posthog From 2aff8a9db7d2d4ac05314b36d9f58f6a5f8f5df3 Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Tue, 14 Sep 2021 08:59:49 -0700 Subject: [PATCH 4/5] changelog --- docs/integrations/sources/posthog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/integrations/sources/posthog.md b/docs/integrations/sources/posthog.md index 931ac7b495c0..a20e70243ea1 100644 --- a/docs/integrations/sources/posthog.md +++ b/docs/integrations/sources/posthog.md @@ -56,6 +56,7 @@ Please follow these [steps](https://posthog.com/docs/api/overview#how-to-obtain- | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.1.4 | 2021-09-14 | [6058](https://github.com/airbytehq/airbyte/pull/6058) | Support self-hosted posthog instances | | 0.1.3 | 2021-07-20 | [4001](https://github.com/airbytehq/airbyte/pull/4001) | Incremental streams read only relevant pages| | 0.1.2 | 2021-07-15 | [4692](https://github.com/airbytehq/airbyte/pull/4692) | Use account information for checking the connection| | 0.1.1 | 2021-07-05 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` env variable for kubernetes support| From a3c43da415b33509c7f0b55283fe08381ebb248d Mon Sep 17 00:00:00 2001 From: Sherif Nada Date: Tue, 14 Sep 2021 17:47:17 -0700 Subject: [PATCH 5/5] conf --- .../connectors/source-posthog/acceptance-test-docker.sh | 2 +- .../source-posthog/integration_tests/invalid_config.json | 2 +- .../connectors/source-posthog/source_posthog/spec.json | 8 ++------ 3 files changed, 4 insertions(+), 8 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh diff --git a/airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh old mode 100644 new mode 100755 index c522eebbd94e..4ceedd9e7ba0 --- a/airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh # Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2):dev # Pull latest acctest image docker pull airbyte/source-acceptance-test:latest diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-posthog/integration_tests/invalid_config.json index 1b3435fb9a6d..2428e75446a3 100644 --- a/airbyte-integrations/connectors/source-posthog/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { "api_key": "value1", - "start_date": "2021-01-01-T00:00:00.000000Z" + "start_date": "2021-01-01T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json b/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json index 6fed07820adb..ae7e8beb9e04 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json @@ -12,9 +12,7 @@ "type": "string", "description": "The date from which you'd like to replicate the data", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": [ - "2021-01-01T00:00:00.000000Z" - ] + "examples": ["2021-01-01T00:00:00Z"] }, "api_key": { "type": "string", @@ -25,9 +23,7 @@ "type": "string", "default": "https://app.posthog.com", "description": "Base PostHog url. Defaults to PostHog Cloud (https://app.posthog.com).", - "examples": [ - "https://posthog.example.com" - ] + "examples": ["https://posthog.example.com"] } } }