diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0f1472b9cf33..664013faee6c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -530,7 +530,7 @@ - name: PostHog sourceDefinitionId: af6d50ee-dddf-4126-a8ee-7faee990774f dockerRepository: airbyte/source-posthog - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/sources/posthog icon: posthog.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index eb4bc5a3d2db..21eaa305330c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5339,7 +5339,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-posthog:0.1.4" +- dockerImage: "airbyte/source-posthog:0.1.5" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/posthog" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-posthog/Dockerfile b/airbyte-integrations/connectors/source-posthog/Dockerfile index bf17d72d4cdb..a79146fc8ed6 100644 --- a/airbyte-integrations/connectors/source-posthog/Dockerfile +++ b/airbyte-integrations/connectors/source-posthog/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-posthog diff --git a/airbyte-integrations/connectors/source-posthog/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-posthog/integration_tests/configured_catalog.json index 0abc1ffecb0f..d992b1c21226 100644 --- a/airbyte-integrations/connectors/source-posthog/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-posthog/integration_tests/configured_catalog.json @@ -45,21 +45,6 @@ "destination_sync_mode": "append", "primary_key": null }, - { - "stream": { - "name": "events_sessions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": null, - "default_cursor_field": null, - "source_defined_primary_key": [["global_session_id"]], - "namespace": null - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null - }, { "stream": { "name": "feature_flags", @@ -75,51 +60,6 @@ "destination_sync_mode": "append", "primary_key": null }, - { - "stream": { - "name": "insights", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": null, - "default_cursor_field": null, - "source_defined_primary_key": [["id"]], - "namespace": null - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null - }, - { - "stream": { - "name": "insights_path", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": null, - "default_cursor_field": null, - "source_defined_primary_key": null, - "namespace": null - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null - }, - { - "stream": { - "name": "insights_sessions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": null, - "default_cursor_field": null, - "source_defined_primary_key": null, - "namespace": null - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null - }, { "stream": { "name": "persons", @@ -134,21 +74,6 @@ "cursor_field": null, "destination_sync_mode": "append", "primary_key": null - }, - { - "stream": { - "name": "trends", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": null, - "default_cursor_field": null, - "source_defined_primary_key": null, - "namespace": null - }, - "sync_mode": "full_refresh", - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null } ] } diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events_sessions.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events_sessions.json deleted file mode 100644 index f655fb947ec4..000000000000 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events_sessions.json +++ /dev/null @@ -1,196 +0,0 @@ -{ - "type": "object", - "properties": { - "distinct_id": { - "type": "string" - }, - "global_session_id": { - "type": "integer" - }, - "length": { - "type": "integer" - }, - "start_time": { - "type": "string", - "format": "date-time" - }, - "end_time": { - "type": "string", - "format": "date-time" - }, - "event_count": { - "type": "integer" - }, - "events": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "distinct_id": { - "type": "string" - }, - "properties": { - "type": "object", - "properties": { - "$os": { - "type": "string" - }, - "$browser": { - "type": "string" - }, - "$device_type": { - "type": "string" - }, - "$current_url": { - "type": "string" - }, - "$host": { - "type": "string" - }, - "$pathname": { - "type": "string" - }, - "$browser_version": { - "type": "integer" - }, - "$screen_height": { - "type": "integer" - }, - "$screen_width": { - "type": "integer" - }, - "$lib": { - "type": "string" - }, - "$lib_version": { - "type": "string" - }, - "$insert_id": { - "type": "string" - }, - "$time": { - "type": "number" - }, - "distinct_id": { - "type": "string" - }, - "$device_id": { - "type": "string" - }, - "$initial_referrer": { - "type": "string" - }, - "$initial_referring_domain": { - "type": "string" - }, - "$referrer": { - "type": "string" - }, - "$referring_domain": { - "type": "string" - }, - "$active_feature_flags": { - "type": "array", - "items": { - "type": "string" - } - }, - "$event_type": { - "type": "string" - }, - "$ce_version": { - "type": "integer" - }, - "token": { - "type": "string" - }, - "$ip": { - "type": "string" - } - } - }, - "event": { - "type": "string" - }, - "timestamp": { - "type": "string", - "format": "date-time" - }, - "person": { - "type": "string" - }, - "elements": { - "type": "array", - "items": { - "type": "object", - "properties": { - "event": { - "type": "string" - }, - "text": { - "type": "string" - }, - "tag_name": { - "type": "string" - }, - "attr_class": { - "type": "string" - }, - "href": { - "type": "string" - }, - "attr_id": { - "type": "string" - }, - "nth_child": { - "type": "integer" - }, - "nth_of_type": { - "type": "integer" - }, - "attributes": { - "type": "object" - }, - "order": { - "type": "integer" - } - } - } - }, - "elements_chain": { - "type": "string" - } - } - } - }, - "properties": { - "type": "object" - }, - "matching_events": { - "type": "array", - "items": { - "type": "string" - } - }, - "email": { - "type": ["null", "string"] - }, - "session_recordings": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "viewed": { - "type": "boolean" - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json deleted file mode 100644 index 7dde76c26253..000000000000 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "name": { - "type": ["string", "null"] - }, - "filters": { - "type": ["object", "null"], - "properties": { - "events": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": ["string", "null"] - }, - "math": { - "type": ["string", "null"] - }, - "name": { - "type": ["string", "null"] - }, - "type": { - "type": ["string", "null"] - }, - "order": { - "type": ["integer", "null"] - }, - "properties": { - "type": "array", - "items": { - "type": ["string", "null", "object"] - } - }, - "math_property": { - "type": ["string", "null"] - } - } - } - }, - "display": { - "type": ["string", "null"] - }, - "filters": { - "type": "array", - "items": { - "type": ["string", "null"] - } - }, - "insight": { - "type": ["string", "null"] - }, - "session": { - "type": ["string", "null"] - }, - "interval": { - "type": ["string", "integer"] - }, - "pagination": { - "type": ["object", "null"] - } - } - }, - "filters_hash": { - "type": ["string", "null"] - }, - "order": { - "type": ["string", "null", "object"] - }, - "deleted": { - "type": "boolean" - }, - "dashboard": { - "type": ["string", "integer", "null"] - }, - "layouts": { - "type": ["object", "null"] - }, - "color": { - "type": ["string", "null"] - }, - "last_refresh": { - "type": ["string", "null"], - "format": "date-time" - }, - "refreshing": { - "type": "boolean" - }, - "result": { - "type": ["string", "null", "object"] - }, - "created_at": { - "type": ["string", "null"], - "format": "date-time" - }, - "saved": { - "type": "boolean" - }, - "created_by": { - "type": ["object", "null"], - "properties": { - "id": { - "type": "integer" - }, - "uuid": { - "type": ["string", "null"] - }, - "distinct_id": { - "type": ["string", "null"] - }, - "first_name": { - "type": ["string", "null"] - }, - "email": { - "type": ["string", "null"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_path.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_path.json deleted file mode 100644 index 15677b9a26a9..000000000000 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_path.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "type": "object", - "properties": { - "source": { - "type": "string" - }, - "source_id": { - "type": "string" - }, - "target": { - "type": "string" - }, - "target_id": { - "type": "string" - }, - "value": { - "type": "integer" - } - } -} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_sessions.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_sessions.json deleted file mode 100644 index c13199f9d299..000000000000 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/insights_sessions.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "type": "object", - "properties": { - "data": { - "type": "array", - "items": { - "type": "number" - } - }, - "labels": { - "type": "array", - "items": { - "type": "string" - } - }, - "days": { - "type": "array", - "items": { - "type": "string" - } - }, - "label": { - "type": "string" - }, - "count": { - "type": "integer" - }, - "chartLabel": { - "type": "string" - } - } -} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/sessions.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/sessions.json deleted file mode 100644 index b9e6251107b6..000000000000 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/sessions.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "type": "object", - "properties": { - "distinct_id": { - "type": "string" - }, - "global_session_id": { - "type": "integer" - }, - "length": { - "type": "integer" - }, - "start_time": { - "type": "string", - "format": "date-time" - }, - "end_time": { - "type": "string", - "format": "date-time" - }, - "event_count": { - "type": "integer" - }, - "events": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "distinct_id": { - "type": "string" - }, - "properties": { - "type": "object", - "properties": { - "$os": { - "type": "string" - }, - "$browser": { - "type": "string" - }, - "$device_type": { - "type": "string" - }, - "$current_url": { - "type": "string" - }, - "$host": { - "type": "string" - }, - "$pathname": { - "type": "string" - }, - "$browser_version": { - "type": "integer" - }, - "$screen_height": { - "type": "integer" - }, - "$screen_width": { - "type": "integer" - }, - "$lib": { - "type": "string" - }, - "$lib_version": { - "type": "string" - }, - "$insert_id": { - "type": "string" - }, - "$time": { - "type": "number" - }, - "distinct_id": { - "type": "string" - }, - "$device_id": { - "type": "string" - }, - "$initial_referrer": { - "type": "string" - }, - "$initial_referring_domain": { - "type": "string" - }, - "$referrer": { - "type": "string" - }, - "$referring_domain": { - "type": "string" - }, - "$active_feature_flags": { - "type": "array", - "items": { - "type": "string" - } - }, - "token": { - "type": "string" - }, - "$ip": { - "type": "string" - } - } - }, - "event": { - "type": "string" - }, - "timestamp": { - "type": "string", - "format": "date-time" - }, - "person": { - "type": "string" - }, - "elements": { - "type": "array", - "items": { - "type": "string" - } - }, - "elements_chain": { - "type": "string" - } - } - } - }, - "properties": { - "type": "object" - }, - "matching_events": { - "type": "array", - "items": { - "type": "string" - } - }, - "email": { - "type": "string" - }, - "session_recordings": { - "type": "array", - "items": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "viewed": { - "type": "boolean" - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/trends.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/trends.json deleted file mode 100644 index 381934a6f624..000000000000 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/trends.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "type": "object", - "properties": { - "action": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "type": { - "type": "string" - }, - "order": { - "type": "string" - }, - "name": { - "type": "string" - }, - "math": { - "type": "string" - }, - "math_property": { - "type": "string" - }, - "properties": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "label": { - "type": "string" - }, - "count": { - "type": "number" - }, - "data": { - "type": "array", - "items": { - "type": "number" - } - }, - "labels": { - "type": "array", - "items": { - "type": "string" - } - }, - "days": { - "type": "array", - "items": { - "type": "string" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/source.py b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py index 38c1673aae48..7c4a5f7b50a0 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/source.py +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/source.py @@ -12,19 +12,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from .streams import ( - Annotations, - Cohorts, - Events, - EventsSessions, - FeatureFlags, - Insights, - InsightsPath, - InsightsSessions, - Persons, - PingMe, - Trends, -) +from .streams import Annotations, Cohorts, Events, FeatureFlags, Persons, PingMe DEFAULT_BASE_URL = "https://app.posthog.com" @@ -46,12 +34,6 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return False, repr(e) def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - event/sessions stream is dynamic. Probably, it contains a list of CURRENT sessions. - In Next day session may expire and wont be available via this endpoint. - So we need a dynamic load data before tests. - This stream was requested to be removed due to this reason. - """ authenticator = TokenAuthenticator(token=config["api_key"]) base_url = config.get("base_url", DEFAULT_BASE_URL) @@ -59,11 +41,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Annotations(authenticator=authenticator, start_date=config["start_date"], base_url=base_url), Cohorts(authenticator=authenticator, base_url=base_url), Events(authenticator=authenticator, start_date=config["start_date"], base_url=base_url), - EventsSessions(authenticator=authenticator, base_url=base_url), FeatureFlags(authenticator=authenticator, base_url=base_url), - Insights(authenticator=authenticator, base_url=base_url), - InsightsPath(authenticator=authenticator, base_url=base_url), - InsightsSessions(authenticator=authenticator, base_url=base_url), Persons(authenticator=authenticator, base_url=base_url), - Trends(authenticator=authenticator, base_url=base_url), ] diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py b/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py index b1b680503e61..a3555c8ac60c 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/streams.py @@ -164,22 +164,6 @@ def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMa return params -class EventsSessions(PosthogStream): - """ - Docs: https://posthog.com/docs/api/events - """ - - primary_key = "global_session_id" - data_field = "result" - - def path(self, **kwargs) -> str: - return "event/sessions" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - resp_json = response.json() - return resp_json.get("pagination") - - class FeatureFlags(PosthogStream): """ Docs: https://posthog.com/docs/api/feature-flags @@ -189,40 +173,6 @@ def path(self, **kwargs) -> str: return "feature_flag" -class Insights(PosthogStream): - """ - Docs: https://posthog.com/docs/api/insights - Endpoint does not support incremental read because id, created_at and last_refresh are ordered in any particular way - """ - - def path(self, **kwargs) -> str: - return "insight" - - -class InsightsPath(PosthogStream): - """ - Docs: https://posthog.com/docs/api/insights - """ - - primary_key = None - data_field = "result" - - def path(self, **kwargs) -> str: - return "insight/path" - - -class InsightsSessions(PosthogStream): - """ - Docs: https://posthog.com/docs/api/insights - """ - - primary_key = None - data_field = "result" - - def path(self, **kwargs) -> str: - return "insight/session" - - class Persons(PosthogStream): """ Docs: https://posthog.com/docs/api/people @@ -232,18 +182,6 @@ def path(self, **kwargs) -> str: return "person" -class Trends(PosthogStream): - """ - Docs: https://posthog.com/docs/api/insights - """ - - primary_key = None - data_field = "result" - - def path(self, **kwargs) -> str: - return "insight/trend" - - class PingMe(PosthogStream): """ Docs: https://posthog.com/docs/api/user diff --git a/docs/integrations/sources/posthog.md b/docs/integrations/sources/posthog.md index fe2bcf5c5286..ba5802156f47 100644 --- a/docs/integrations/sources/posthog.md +++ b/docs/integrations/sources/posthog.md @@ -55,6 +55,7 @@ Please follow these [steps](https://posthog.com/docs/api/overview#how-to-obtain- | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.5 | 2021-12-24 | [9082](https://github.com/airbytehq/airbyte/pull/9082) | Remove obsolete session_events and insights streams | | 0.1.4 | 2021-09-14 | [6058](https://github.com/airbytehq/airbyte/pull/6058) | Support self-hosted posthog instances | | 0.1.3 | 2021-07-20 | [4001](https://github.com/airbytehq/airbyte/pull/4001) | Incremental streams read only relevant pages | | 0.1.2 | 2021-07-15 | [4692](https://github.com/airbytehq/airbyte/pull/4692) | Use account information for checking the connection |